From 31a0c38be6e2f506e669e57dc30607a9f87dcc5b Mon Sep 17 00:00:00 2001 From: ajnasnb Date: Sat, 8 Aug 2026 01:29:58 +0530 Subject: [PATCH 01/22] release: publish Qarinah 0.1.6 research evidence --- .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- .github/workflows/ci.yml | 2 + .github/workflows/deploy-site.yml | 157 + .github/workflows/publish-npm.yml | 18 +- .gitignore | 2 + CONTRIBUTING.md | 10 + README.md | 12 +- bench/research/swe-bench-lite.mjs | 61 +- bench/results/benchmark-release-0.1.6.json | 232 + bench/results/continuation-context-0.1.6.json | 46 + bench/results/multifile-context-0.1.6.json | 5975 + .../research-retrieval-development-v0.4.json | 192009 +++++++++++++++ docs/API-REFERENCE.md | 8 +- docs/BENCHMARKS.md | 34 +- docs/CROSS-SESSION-CONTINUATION-BENCHMARK.md | 8 +- docs/ECOSYSTEM-LAUNCH.md | 2 +- docs/FAQ.md | 4 +- docs/GETTING-STARTED.md | 2 +- docs/HOST-INTEGRATIONS.md | 12 +- docs/MCP-GUIDE.md | 14 +- docs/RECIPES.md | 4 +- docs/RESEARCH-BENCHMARK.md | 29 +- docs/RESEARCH-DEVELOPMENT-RESULTS-v0.3.md | 2 + docs/RESEARCH-DEVELOPMENT-RESULTS-v0.4.md | 49 + docs/ROADMAP.md | 2 +- docs/TOKEN-EFFICIENT-CONTEXT.md | 4 +- docs/WHITEPAPER.md | 44 +- .../claude/qarinah/.claude-plugin/plugin.json | 2 +- .../claude/qarinah/runtime/qarinah.mjs | 4 +- .../codex/qarinah/.codex-plugin/plugin.json | 2 +- .../codex/qarinah/runtime/qarinah.mjs | 4 +- ...inah-Technical-White-Paper-v1.3.build.json | 53 + .../Qarinah-Technical-White-Paper-v1.3.pdf | Bin 0 -> 260326 bytes ...inah-Technical-White-Paper-v1.3.pdf.sha256 | 1 + ...h-Technical-White-Paper-v1.3.source.sha256 | 1 + package-lock.json | 4 +- package.json | 13 +- scripts/build-site.mjs | 37 +- scripts/build-whitepaper-pdf-v1.3.py | 229 + scripts/build-whitepaper-pdf.py | 2 +- scripts/check-docs.mjs | 106 +- scripts/check-site.mjs | 11 +- scripts/continuation-evidence-lib.mjs | 41 +- scripts/evaluate-research-retrieval-v0.2.mjs | 166 +- scripts/evaluate-research-retrieval-v0.4.mjs | 2 + scripts/verify-benchmark-release-0.1.6.mjs | 245 + .../verify-codex-continuation-evidence.mjs | 43 +- .../verify-research-development-evidence.mjs | 26 +- scripts/verify-research-evidence.mjs | 2 +- scripts/verify-research-sufficiency-v0.3.mjs | 6 +- server.json | 4 +- src/version.js | 2 +- test/deploy-workflow.test.js | 61 + types/index.d.ts | 2 +- website/static/llms-full.txt | 5 +- website/static/llms.txt | 4 +- wrangler.jsonc | 9 + 57 files changed, 199659 insertions(+), 172 deletions(-) create mode 100644 .github/workflows/deploy-site.yml create mode 100644 bench/results/benchmark-release-0.1.6.json create mode 100644 bench/results/continuation-context-0.1.6.json create mode 100644 bench/results/multifile-context-0.1.6.json create mode 100644 bench/results/research-retrieval-development-v0.4.json create mode 100644 docs/RESEARCH-DEVELOPMENT-RESULTS-v0.4.md create mode 100644 output/pdf/Qarinah-Technical-White-Paper-v1.3.build.json create mode 100644 output/pdf/Qarinah-Technical-White-Paper-v1.3.pdf create mode 100644 output/pdf/Qarinah-Technical-White-Paper-v1.3.pdf.sha256 create mode 100644 output/pdf/Qarinah-Technical-White-Paper-v1.3.source.sha256 create mode 100644 scripts/build-whitepaper-pdf-v1.3.py create mode 100644 scripts/evaluate-research-retrieval-v0.4.mjs create mode 100644 scripts/verify-benchmark-release-0.1.6.mjs create mode 100644 test/deploy-workflow.test.js create mode 100644 wrangler.jsonc diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 12967b1..178722e 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -13,7 +13,7 @@ body: attributes: label: Qarinah version description: Output of `npm view qarinah version` and the installed version. - placeholder: "0.1.3" + placeholder: "0.1.6" validations: required: true - type: dropdown diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b4830c2..877a9d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,6 +53,8 @@ jobs: node: [22, 24, 26] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: ${{ matrix.node }} diff --git a/.github/workflows/deploy-site.yml b/.github/workflows/deploy-site.yml new file mode 100644 index 0000000..4311ab0 --- /dev/null +++ b/.github/workflows/deploy-site.yml @@ -0,0 +1,157 @@ +name: Deploy website + +on: + workflow_dispatch: + inputs: + version: + description: Exact published qarinah version to deploy + required: true + type: string + expected_git_commit: + description: Exact reviewed lowercase 40-character main commit + required: true + type: string + confirmation: + description: Type deploy qarinah@VERSION + required: true + type: string + +permissions: + contents: read + actions: read + +concurrency: + group: deploy-qarinah-site + cancel-in-progress: false + +jobs: + deploy: + name: Build, verify, and deploy exact published site + runs-on: ubuntu-latest + timeout-minutes: 30 + environment: + name: website + url: https://qarinah.io + env: + DEPLOY_COMMIT: ${{ inputs.expected_git_commit }} + EXPECTED_MANUAL_VERSION: ${{ inputs.version }} + steps: + - name: Validate trusted trigger + shell: bash + env: + CONFIRMATION: ${{ inputs.confirmation }} + EXPECTED_GIT_COMMIT: ${{ inputs.expected_git_commit }} + run: | + set -euo pipefail + [[ "${GITHUB_REPOSITORY}" == "AjnasNB/qarinah" ]] || { echo "Unexpected repository." >&2; exit 1; } + [[ "${DEPLOY_COMMIT}" =~ ^[0-9a-f]{40}$ ]] || { echo "Invalid deployment commit." >&2; exit 1; } + [[ "${GITHUB_REF}" == "refs/heads/main" ]] || { echo "Manual deployment must run from main." >&2; exit 1; } + [[ "${EXPECTED_GIT_COMMIT}" == "${GITHUB_SHA}" ]] || { echo "Manual deployment commit is not the exact selected main commit." >&2; exit 1; } + [[ "${CONFIRMATION}" == "deploy qarinah@${EXPECTED_MANUAL_VERSION}" ]] || { echo "Manual confirmation mismatch." >&2; exit 1; } + + - name: Check out the exact published commit + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ env.DEPLOY_COMMIT }} + fetch-depth: 0 + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 24.15.0 + registry-url: https://registry.npmjs.org + package-manager-cache: false + + - name: Verify checked-out identity + shell: bash + run: | + set -euo pipefail + actual_commit="$(git rev-parse HEAD)" + [[ "${actual_commit}" == "${DEPLOY_COMMIT}" ]] || { echo "Checked-out commit mismatch." >&2; exit 1; } + package_name="$(node -p 'require("./package.json").name')" + package_version="$(node -p 'require("./package.json").version')" + [[ "${package_name}" == "qarinah" ]] || { echo "Unexpected package name." >&2; exit 1; } + [[ "${package_version}" == "${EXPECTED_MANUAL_VERSION}" ]] || { echo "Manual version does not match the reviewed commit." >&2; exit 1; } + echo "PACKAGE_VERSION=${package_version}" >> "${GITHUB_ENV}" + + - name: Require a successful trusted publish + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + node --input-type=module <<'NODE' + const [owner, repository] = process.env.GITHUB_REPOSITORY.split("/"); + const url = new URL(`https://api.github.com/repos/${owner}/${repository}/actions/workflows/publish-npm.yml/runs`); + url.searchParams.set("event", "workflow_dispatch"); + url.searchParams.set("status", "success"); + url.searchParams.set("head_sha", process.env.DEPLOY_COMMIT); + url.searchParams.set("per_page", "10"); + + const response = await fetch(url, { + headers: { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${process.env.GH_TOKEN}`, + "X-GitHub-Api-Version": "2022-11-28" + } + }); + if (!response.ok) throw new Error(`Unable to verify trusted publish: GitHub returned ${response.status}.`); + const payload = await response.json(); + const trusted = payload.workflow_runs?.some((run) => + run.head_sha === process.env.DEPLOY_COMMIT + && run.head_branch === "main" + && run.event === "workflow_dispatch" + && run.conclusion === "success" + && (run.path === ".github/workflows/publish-npm.yml" + || run.path?.startsWith(".github/workflows/publish-npm.yml@")) + ); + if (!trusted) throw new Error("No successful trusted publish exists for the exact deployment commit."); + NODE + + - name: Wait for the exact npm package version + shell: bash + run: | + set -euo pipefail + retry_delay_seconds() { + local attempt="$1" + local delay=$((2 ** (attempt - 1))) + if (( delay > 30 )); then delay=30; fi + printf '%s' "${delay}" + } + + published_version="" + for attempt in {1..30}; do + published_version="$(npm view "qarinah@${PACKAGE_VERSION}" version 2>/dev/null || true)" + if [[ "${published_version}" == "${PACKAGE_VERSION}" ]]; then + echo "qarinah@${PACKAGE_VERSION} is published; website verification may proceed." + break + fi + if (( attempt < 30 )); then + delay="$(retry_delay_seconds "${attempt}")" + echo "qarinah@${PACKAGE_VERSION} is not visible yet (attempt ${attempt}/30); retrying in ${delay}s." + sleep "${delay}" + fi + done + if [[ "${published_version}" != "${PACKAGE_VERSION}" ]]; then + echo "qarinah@${PACKAGE_VERSION} was not available after 30 attempts; refusing to build or deploy the site." >&2 + exit 1 + fi + + - name: Install exact dependencies without lifecycle scripts + run: npm ci --ignore-scripts + + - name: Build site-dist + run: npm run build:site + + - name: Verify site-dist + run: npm run check:site + + - name: Deploy checked assets + uses: cloudflare/wrangler-action@ebbaa1584979971c8614a24965b4405ff95890e0 # v4.0.0 + with: + apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} + accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + wranglerVersion: "4.120.0" + packageManager: npm + command: deploy --config wrangler.jsonc diff --git a/.github/workflows/publish-npm.yml b/.github/workflows/publish-npm.yml index 927201f..b4e8c62 100644 --- a/.github/workflows/publish-npm.yml +++ b/.github/workflows/publish-npm.yml @@ -6,7 +6,7 @@ on: version: description: Exact package version to publish required: true - default: 0.1.3 + default: 0.1.6 type: string expected_git_commit: description: Exact reviewed lowercase 40-character commit SHA @@ -79,6 +79,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.sha }} + fetch-depth: 0 persist-credentials: false - name: Set up Node.js uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 @@ -203,4 +204,17 @@ jobs: npm init --yes >/dev/null npm install "qarinah@${EXPECTED_VERSION}" --ignore-scripts node --input-type=module -e "await import('qarinah'); await import('qarinah/codex'); await import('qarinah/claude'); await import('qarinah/mcp')" - npm audit signatures + verified="" + for attempt in {1..10}; do + if npm audit signatures; then + verified="yes" + break + fi + if (( attempt < 10 )); then + delay=$((2 ** (attempt - 1))) + if (( delay > 30 )); then delay=30; fi + echo "Registry signature verification was not ready (attempt ${attempt}/10); retrying in ${delay}s." + sleep "${delay}" + fi + done + [[ "${verified}" == "yes" ]] || { echo "Registry signature verification did not succeed." >&2; exit 1; } diff --git a/.gitignore b/.gitignore index abbee30..ae581b7 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ benchmark-results/ *.tgz site-dist/ tmp/ +__pycache__/ +*.pyc diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 86628b0..b6ffc7e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,6 +24,16 @@ npm run check Run focused tests while iterating, then run `npm run check` before requesting review. A pull request is not release-ready until the full check passes from a clean install. +### White-paper builds + +White-paper generation is a repository-maintainer workflow, not a public npm-package command. The Python builders and generated PDF are intentionally excluded from the npm tarball. From a complete source checkout with Python 3 and ReportLab installed, build the current paper directly: + +```sh +python scripts/build-whitepaper-pdf-v1.3.py +``` + +The v1.3 source receipt binds `docs/WHITEPAPER.md`, the shared layout engine, and the version-specific wrapper. Its companion build metadata records the Python, ReportLab, platform, and font inputs used for that generated artifact. Never run the historical builder directly to replace a published PDF. + ## Architecture invariants Qarinah's JSONL event log is the authoritative record. Graphs, indexes, Markdown views, OKF exports, and context packs must remain deterministic derivatives. diff --git a/README.md b/README.md index 5679a81..963c568 100644 --- a/README.md +++ b/README.md @@ -103,8 +103,8 @@ The JSONL chain remains authoritative. Graph, index, Markdown, dashboard, and OK

Technical paper ·  - Publication PDF ·  - Zenodo record ·  + White-paper v1.3 PDF ·  + White-paper v1.3 DOI ·  Architecture ·  Dashboard ·  Benchmarks ·  @@ -359,15 +359,15 @@ The repository also runs `npm run mcp:smoke` against the exact bundled Codex and ### Install once, initialize each project -Install the reviewed `v0.1.5` plugin once in each host: +Install the reviewed `v0.1.6` plugin once in each host after that release is published: ```sh # Codex: personal installation, available to opted-in projects. -codex plugin marketplace add AjnasNB/qarinah --ref v0.1.5 +codex plugin marketplace add AjnasNB/qarinah --ref v0.1.6 codex plugin add qarinah@qarinah # Claude Code: personal installation across projects. -claude plugin marketplace add AjnasNB/qarinah@v0.1.5 --scope user +claude plugin marketplace add AjnasNB/qarinah@v0.1.6 --scope user claude plugin install qarinah@qarinah --scope user ``` @@ -461,7 +461,7 @@ The long-document evaluator adds a fixed 600-token ceiling over a deterministic The [cross-session continuation benchmark](docs/CROSS-SESSION-CONTINUATION-BENCHMARK.md) adds a 42-record two-session fixture for context summarization, evidence links, and fresh-session retrieval. Its complete 1,039-token cited audit pack is 89.05% smaller than the 9,489-token full-ledger estimate and preserves all three summary source IDs and hashes. A separate 119-token model-facing capsule points to that verified pack and the selected summary event, reaching 98.75% reduction on the same unchanged fixture without removing the audit trail. The read also leaves deliberately stale derived state unchanged. A separate provider-backed Codex-to-Codex smoke uses distinct ephemeral sessions with native resume disabled, requires the second session to query Qarinah and cite its evidence, and verifies the resulting patch with tests. The provider smoke is product evidence, not a controlled research result. -The separate [real-repository research track](docs/RESEARCH-BENCHMARK.md) pins 300 public SWE-bench Lite tasks into a chronological 60-task warm-up / 240-task development split. Frozen exploratory v0.1 found that BM25 beat the original balanced Qarinah ranker. Admission-first v2 now preserves admitted BM25 ranking while retaining repository, temporal, retention, disclosure, conflict, supersession, provenance, and budget controls; online MRR improves from 0.601 to 0.696 against balanced-v1 under the graded structural development oracle. Graph ranking adds no measured value here. At the conservative v0.3 operating point, the run observed 0/49 static and 0/31 online direct false accepts by abstaining aggressively; exact 95% upper bounds remain 7.25% and 11.22%, and coverage is only 3.33%-5.00%. The [latest development result](docs/RESEARCH-DEVELOPMENT-RESULTS-v0.3.md) also freezes 387 positive tasks, 20 abstention controls, a contamination audit, and a pre-outcome 40-pair power check. This phase does not measure coding-agent task success or provider usage. +The separate [real-repository research track](docs/RESEARCH-BENCHMARK.md) pins 300 public SWE-bench Lite tasks into a chronological 60-task warm-up / 240-task development split. Frozen exploratory v0.1 found that BM25 beat the original balanced Qarinah ranker. Admission-first v2 now preserves admitted BM25 ranking while retaining repository, temporal, retention, disclosure, conflict, supersession, provenance, and budget controls; online MRR improves from 0.601 to 0.696 against balanced-v1 under the graded structural development oracle. Graph ranking adds no measured value here. Historical v0.3 calibrated a conservative decision over frozen v0.2 scores. The [production-bound v0.4 recomputation](docs/RESEARCH-DEVELOPMENT-RESULTS-v0.4.md) uses current `evidence-sufficiency-v2`: it observed 10/10 static and 15/15 online direct accepts as structural-oracle positives, with 0/49 and 0/31 false accepts. Exact 95% false-acceptance upper bounds remain 7.25% and 11.22%, and coverage remains deliberately low at 4.17%-6.25%. The research package also freezes 387 positive tasks, 20 abstention controls, a contamination audit, and a pre-outcome 40-pair power check. This phase does not measure coding-agent task success or provider usage. ## License and ownership diff --git a/bench/research/swe-bench-lite.mjs b/bench/research/swe-bench-lite.mjs index 00fb991..062eaa4 100644 --- a/bench/research/swe-bench-lite.mjs +++ b/bench/research/swe-bench-lite.mjs @@ -1,4 +1,5 @@ import { createHash } from "node:crypto"; +import { setTimeout as delay } from "node:timers/promises"; export const DATASET_ID = "princeton-nlp/SWE-bench_Lite"; export const DATASET_REVISION = "6ec7bb89b9342f664a54a6e0a6ea6501d3437cc2"; @@ -59,22 +60,48 @@ function requestUrl(base, parameters) { return url; } +async function withTransientRetries(label, operation) { + const attempts = 4; + let lastError; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + try { + return await operation(); + } catch (error) { + lastError = error; + if (error?.retryable === false || attempt === attempts) break; + await delay(250 * (2 ** (attempt - 1))); + } + } + throw new Error(`${label} failed after ${attempts} attempts.`, { cause: lastError }); +} + +function requireSuccessfulResponse(response, label) { + if (response.ok) return; + const error = new Error(`${label} failed with HTTP ${response.status}.`); + error.retryable = response.status === 408 || response.status === 429 || response.status >= 500; + throw error; +} + async function getJson(url, label) { - const response = await fetch(url, { - headers: { "user-agent": "qarinah-research-benchmark/1" }, - signal: AbortSignal.timeout(30_000) + return withTransientRetries(label, async () => { + const response = await fetch(url, { + headers: { "user-agent": "qarinah-research-benchmark/1" }, + signal: AbortSignal.timeout(30_000) + }); + requireSuccessfulResponse(response, label); + return response.json(); }); - if (!response.ok) throw new Error(`${label} failed with HTTP ${response.status}.`); - return response.json(); } async function getBytes(url, label) { - const response = await fetch(url, { - headers: { "user-agent": "qarinah-research-benchmark/2" }, - signal: AbortSignal.timeout(60_000) + return withTransientRetries(label, async () => { + const response = await fetch(url, { + headers: { "user-agent": "qarinah-research-benchmark/2" }, + signal: AbortSignal.timeout(60_000) + }); + requireSuccessfulResponse(response, label); + return Buffer.from(await response.arrayBuffer()); }); - if (!response.ok) throw new Error(`${label} failed with HTTP ${response.status}.`); - return Buffer.from(await response.arrayBuffer()); } export async function fetchDatasetMetadata() { @@ -448,11 +475,21 @@ export function buildRepositoryManifest(rows, metadata, sourceArtifact, historic return { ...manifest, content_sha256: sha256(JSON.stringify(manifest)) }; } -export async function loadPinnedDevelopmentDataset() { +export async function loadPinnedDevelopmentDataset(options = {}) { + const suppliedSourceArtifact = options.sourceArtifact ?? null; + if (suppliedSourceArtifact !== null) { + if (suppliedSourceArtifact.path !== DATASET_TEST_ARTIFACT + || !/^sha256:[0-9a-f]{64}$/u.test(suppliedSourceArtifact.sha256) + || !Number.isSafeInteger(suppliedSourceArtifact.bytes) + || suppliedSourceArtifact.bytes < 1 + || typeof suppliedSourceArtifact.url !== "string") { + throw new TypeError("sourceArtifact must be exact pinned-artifact metadata from the committed development corpus."); + } + } const [metadata, rows, sourceArtifact] = await Promise.all([ fetchDatasetMetadata(), fetchDatasetRows(), - fetchDatasetArtifactMetadata() + suppliedSourceArtifact === null ? fetchDatasetArtifactMetadata() : suppliedSourceArtifact ]); return { metadata, diff --git a/bench/results/benchmark-release-0.1.6.json b/bench/results/benchmark-release-0.1.6.json new file mode 100644 index 0000000..21f9ccd --- /dev/null +++ b/bench/results/benchmark-release-0.1.6.json @@ -0,0 +1,232 @@ +{ + "schemaVersion": "qarinah.benchmark-release.v2", + "packageVersion": "0.1.6", + "paperVersion": "1.3", + "status": "release-candidate-locally-verified-not-published", + "preparedAt": "2026-08-08", + "receiptScope": "timestamped-pre-publication-local-verification", + "lifecycleSemantics": "These status fields record the state when the exact release source was locally verified; they are not a live publication-status endpoint.", + "paperArtifact": { + "status": "local-release-candidate-zenodo-version-doi-reserved", + "conceptDoi": "10.5281/zenodo.21547684", + "newVersionDoi": "10.5281/zenodo.21843240", + "zenodoDraftRecordId": "21843240", + "doiStatus": "reserved-draft-not-registered-or-published", + "path": "output/pdf/Qarinah-Technical-White-Paper-v1.3.pdf", + "sha256": "sha256:4b0c87adc03504dafa9817e3cf6c0fc78c02f7dd2c8d9951e94ae9d722f8ec20", + "sourceReceiptPath": "output/pdf/Qarinah-Technical-White-Paper-v1.3.source.sha256", + "sourceReceiptSha256": "sha256:9bb16836c87beb3b47d667577160fffab5a8092a9682ae9a379909ec180fd885", + "pdfReceiptPath": "output/pdf/Qarinah-Technical-White-Paper-v1.3.pdf.sha256", + "pdfReceiptSha256": "sha256:5afbd34142936d4d93b52b703690e60e5a88482da69d70f3c019bbdac526cfad", + "buildMetadataPath": "output/pdf/Qarinah-Technical-White-Paper-v1.3.build.json", + "buildMetadataSha256": "sha256:a9b306c9e2461f585d8f5fbd615aa209eb19e8a0926490318f2d7fc4ee4e29bf" + }, + "portableTokenEstimator": { + "method": "ceil(characters / 4)", + "exactProviderTokenizer": false, + "providerBillingMeasurement": false + }, + "headlineContextResults": [ + { + "id": "six-task-repeated-context", + "baselineEstimatedTokens": 442113, + "qarinahEstimatedTokens": 5682, + "exactReduction": 0.987148, + "displayReduction": "98.7148%", + "coverage": "Every required target directly covered in the top five.", + "artifact": { + "path": "bench/results/software-task-context-0.1.0.json", + "sha256": "sha256:92ab5041c609caf226729724555bd9b70c0a52cf2e2418370c228ea6a9452594" + } + }, + { + "id": "two-session-model-facing-capsule", + "baselineEstimatedTokens": 9489, + "qarinahEstimatedTokens": 119, + "exactReduction": 0.987459, + "displayReduction": "98.75%", + "coverage": "Summary event ID/hash and complete audit-pack manifest pointer retained.", + "artifact": { + "path": "bench/results/continuation-context-0.1.6.json", + "sha256": "sha256:3e0d7388bfb65a68da4e533d22cf02aefeb73b450c751fe85c38fba7ccf51e19" + } + }, + { + "id": "two-session-complete-audit-pack", + "baselineEstimatedTokens": 9489, + "qarinahEstimatedTokens": 1039, + "exactReduction": 0.890505, + "displayReduction": "89.05%", + "coverage": "All three summary-source event IDs/hashes plus a selected raw source retained.", + "artifact": { + "path": "bench/results/continuation-context-0.1.6.json", + "sha256": "sha256:3e0d7388bfb65a68da4e533d22cf02aefeb73b450c751fe85c38fba7ccf51e19" + } + } + ], + "multiFileProjectStudy": { + "status": "deterministic-synthetic-local-retrieval-and-projection-integrity", + "totalFiles": 190, + "positiveQueriesPassed": 380, + "unsupportedQueriesCorrectlyRejected": 9, + "allScalesPassed": true, + "providerModelCalls": 0, + "providerBillingMeasurement": false, + "scales": [ + { + "fileCount": 40, + "positiveQueriesPassed": 80, + "minimumEstimatedReduction": 0.934803 + }, + { + "fileCount": 50, + "positiveQueriesPassed": 100, + "minimumEstimatedReduction": 0.946948 + }, + { + "fileCount": 100, + "positiveQueriesPassed": 200, + "minimumEstimatedReduction": 0.971521 + } + ], + "artifact": { + "path": "bench/results/multifile-context-0.1.6.json", + "sha256": "sha256:d8d6ff65e1e2420c13bf6094d1cd7307499b4a422dfa28cff3baf31dd4f25b99" + } + }, + "realRepositoryDevelopmentStudy": { + "status": "development-after-exploratory-inspection-not-confirmatory", + "providerModelCalls": 0, + "officialPatchResolutionExecuted": false, + "dataset": { + "name": "princeton-nlp/SWE-bench_Lite", + "revision": "6ec7bb89b9342f664a54a6e0a6ea6501d3437cc2", + "rawTestParquetSha256": "sha256:7a21f37b8bc179c7db5beeb14e88ac538ba283455c776e6b2535bbfb6e3551b4", + "tasks": 300, + "exactRepositoryIdentifiers": 12, + "warmupTasks": 60, + "heldoutQueries": 240 + }, + "onlinePrequential": { + "scorableQueries": 209, + "qarinahV2RecallAt10": 0.538341, + "qarinahV2MeanReciprocalRank": 0.695595, + "balancedV1MeanReciprocalRank": 0.600665, + "pairedMrrDifference": 0.09493, + "repositoryClusteredBootstrap95": [0.057183, 0.111456], + "graphRankingImprovement": 0 + }, + "productionBoundEvidenceGate": { + "method": "evidence-sufficiency-v2", + "static": { + "acceptedDirect": 10, + "queries": 240, + "observedFalseAccepts": 0, + "structuralOracleNegativeQueries": 49, + "acceptanceCoverage": 0.041667, + "acceptedPrecisionExact95Lower": 0.691503, + "falseAcceptanceRateExact95Upper": 0.072519 + }, + "onlinePrequential": { + "acceptedDirect": 15, + "queries": 240, + "observedFalseAccepts": 0, + "structuralOracleNegativeQueries": 31, + "acceptanceCoverage": 0.0625, + "acceptedPrecisionExact95Lower": 0.781981, + "falseAcceptanceRateExact95Upper": 0.112189 + }, + "humanValidatedRelevance": false + }, + "artifacts": [ + { + "role": "current-production-bound-development-v0.4", + "path": "bench/results/research-retrieval-development-v0.4.json", + "sha256": "sha256:607359a947e7a849512d3fcb588bc88c2b34e1289f15b735a2de0c3895a21a18" + }, + { + "role": "historical-development-v0.2", + "path": "bench/results/research-retrieval-development-v0.2.json", + "sha256": "sha256:bfe8015811ffbecd5e3c00eb9f4a1e104478605cd605442a1ec96d67582e4b3f" + }, + { + "role": "historical-threshold-calibration-v0.3", + "path": "bench/results/research-sufficiency-development-v0.3.json", + "sha256": "sha256:fda449d36c6de2d2aea3d5cb8b69ceac2c57dfcb1c0b63161208a6ec1fc28986" + } + ] + }, + "providerBackedProductSmoke": { + "currentReleaseReceiptPresent": false, + "currentReleaseProviderCalls": 0, + "currentReleaseClaimEligible": false, + "historicalReceipt": { + "classification": "historical-0.1.5-provider-backed-product-smoke-not-controlled-research", + "packageVersion": "0.1.5", + "artifact": { + "path": "bench/results/codex-cross-session-continuation-0.1.5.json", + "sha256": "sha256:c6ebb38626550d1cb9293a42f7dda292eb55558865d53c735b7996414752783f" + }, + "distinctSessions": true, + "nativeResumeUsed": false, + "queriedQarinahBeforeContinuation": true, + "acceptanceTestsPassed": true + } + }, + "frozenConfirmatoryProtocol": { + "status": "protocol-frozen-results-unobserved", + "sourceDataset": "princeton-nlp/SWE-bench_Verified", + "sourceTasks": 500, + "eligibleRetrievalTasks": 387, + "pairedAgentTasks": 40, + "abstentionControls": 20, + "officialContainerEvaluationPlanned": true, + "providerUsageReceiptsRequired": true, + "resultsObserved": false, + "artifacts": [ + { + "path": "bench/final/final-task-manifest-v1.json", + "sha256": "sha256:eca1e32961cea6979a6c488cf5c7af19b2247923934059986f2e087d07016e54" + }, + { + "path": "bench/final/final-abstention-controls-v1.json", + "sha256": "sha256:367730f076d5d41aa7ef516aba57c04ff677fc172d97e247ff9fb3b896585990" + }, + { + "path": "bench/final/contamination-audit-v1.json", + "sha256": "sha256:d4692b9b88147e9ef75708ad2815f02285ba91abeb5bc4f01f567d303ddc26bf" + } + ] + }, + "citations": [ + { + "id": "swe-bench-iclr-2024", + "url": "https://openreview.net/forum?id=VTF8yNQM66" + }, + { + "id": "swe-bench-lite-pinned-dataset", + "url": "https://huggingface.co/datasets/princeton-nlp/SWE-bench_Lite" + }, + { + "id": "bm25-okapi-trec-3", + "url": "https://doi.org/10.6028/NIST.SP.500-225.routing-city" + }, + { + "id": "longmemeval-iclr-2025", + "url": "https://proceedings.iclr.cc/paper_files/paper/2025/file/d813d324dbf0598bbdc9c8e79740ed01-Paper-Conference.pdf" + }, + { + "id": "swe-bench-verified", + "url": "https://www.swebench.com/verified.html" + } + ], + "claimBoundary": [ + "The 0.1.6 package, website, and v1.3 paper are locally prepared release candidates; DOI 10.5281/zenodo.21843240 is reserved for a Zenodo draft and is not registered or published until that draft is published.", + "The three headline percentages are portable estimated context-volume reductions, not provider billing reductions.", + "The multi-file result is a deterministic synthetic local retrieval and projection-integrity regression, not provider task-completion evidence.", + "The real-repository results evaluate retrieval and evidence admission on inspected development data, not generated-patch resolution.", + "The only provider-backed continuation receipt is historical Qarinah 0.1.5 evidence and is not a 0.1.6 result.", + "The 40-task provider-backed confirmatory protocol has no observed outcome in Qarinah 0.1.6.", + "No result is a universal guarantee across repositories, models, languages, or tasks." + ] +} diff --git a/bench/results/continuation-context-0.1.6.json b/bench/results/continuation-context-0.1.6.json new file mode 100644 index 0000000..f132f56 --- /dev/null +++ b/bench/results/continuation-context-0.1.6.json @@ -0,0 +1,46 @@ +{ + "schemaVersion": "qarinah.continuation-context-eval-result.v1", + "packageVersion": "0.1.6", + "fixture": { + "description": "Deterministic two-session fixture with an inferred handoff summary linked to extracted prompt, verified test outcome, and completed-turn evidence, followed by lifecycle events that deliberately stale persisted derived state.", + "tokenEstimator": "portable ceil(characters / 4)", + "providerBillingMeasurement": false + }, + "expected": { + "records": 42, + "sourceEventCount": 3, + "summaryRank": 3, + "summaryConfidence": "inferred", + "summarySelected": true, + "sourceIdsPreserved": true, + "sourceHashesPreserved": true, + "sourceEventsSelected": 1, + "distinctSessionIds": 2, + "stalePersistedHeadDetected": true, + "staleReadRecoveredInMemory": true, + "persistedDerivedStateUnchanged": true, + "citationsValid": true, + "coverage": "partial", + "evidenceState": "DIRECTLY_SUPPORTED", + "rankingProfile": "admission-first-v2", + "temporalBoundary": "strict-before", + "rawHistoryTokens": 9489, + "packTokens": 1039, + "estimatedTokenReduction": 0.890505, + "capsuleTokens": 119, + "capsuleEstimatedTokenReduction": 0.987459, + "capsuleSummaryEventLinked": true, + "capsuleManifestLinked": true, + "capsuleSourceLinksAuditable": true, + "capsuleUntrustedBoundary": true, + "capsuleTruncated": false, + "doctorOk": true + }, + "claim": "A fresh session recovered a compact evidence-linked handoff from the verified ledger after lifecycle capture advanced the head. The complete audit pack retained every summary source ID and hash; its model-facing handoff capsule retained the selected summary event ID/hash and pack manifest pointer while the zero-write read left persisted derived state unchanged.", + "limitations": [ + "This deterministic fixture measures retrieval and evidence linkage, not provider task quality.", + "The summary is inferred and remains subordinate to its cited source events.", + "The compact handoff capsule points to the complete audited pack and selected summary instead of embedding every raw source citation in the model-facing text.", + "The portable token estimate is not a provider bill." + ] +} diff --git a/bench/results/multifile-context-0.1.6.json b/bench/results/multifile-context-0.1.6.json new file mode 100644 index 0000000..2520413 --- /dev/null +++ b/bench/results/multifile-context-0.1.6.json @@ -0,0 +1,5975 @@ +{ + "schemaVersion": "qarinah.multifile-context-eval-result.v1", + "packageVersion": "0.1.6", + "fixture": { + "description": "Deterministic 40/50/100-file project workspaces with nested JavaScript and Markdown files, resolved imports/links, one unique answer-bearing memory record per file, lexical distractors, graph-only evidence, supersession, contradiction, stale derived projections, and unsupported controls.", + "fileCounts": [ + 40, + 50, + 100 + ], + "totalFiles": 190, + "totalPositiveQueries": 380, + "totalUnsupportedQueries": 9, + "tokenEstimator": "portable ceil(characters / 4)", + "providerBillingMeasurement": false + }, + "expected": { + "allScalesPassed": true, + "scales": [ + { + "fileCount": 40, + "eventCount": 45, + "exactQueries": 40, + "typoTolerantQueries": 40, + "unsupportedQueries": 3, + "allTargetsRankedFirst": true, + "allAnswersPreserved": true, + "allExactQueriesUsedSqlite": true, + "allTypoQueriesUsedFuzzyRetrieval": true, + "allExactQueriesAcceptedDirect": true, + "allTypoQueriesConservativelyAbstained": true, + "persistedInMemoryParity": true, + "projectStructure": { + "filesScanned": 40, + "graphFileNodes": 40, + "graphReferenceEdges": 40, + "latePathAndReferencePreserved": true, + "markdownFirstMiddleLastPathsPreserved": true + }, + "graphLinkedDecisionRecovered": true, + "supersededDecisionExcluded": true, + "conflictVisible": true, + "sqliteReadModel": { + "schemaVersion": 1, + "eventCountMatchesLedger": true, + "headMatchesLedger": true, + "requiredTablesPresent": true, + "lateFileEvidenceFound": true + }, + "derivedRepair": { + "staleGraphDetectedAndRebuilt": true, + "staleMarkdownDetectedAndRebuilt": true, + "repairedRetrievalRank": 1 + }, + "correctAbstention": true, + "abstention": [ + { + "query": "unsupported40 nebula authorization secret", + "correctAbstention": true, + "errorCode": "CONTEXT_COVERAGE_TOO_LOW" + }, + { + "query": "missing40 lunar rollback credential", + "correctAbstention": true, + "errorCode": "CONTEXT_COVERAGE_TOO_LOW" + }, + { + "query": "absent40 oceanic encryption answer", + "correctAbstention": true, + "errorCode": "CONTEXT_COVERAGE_TOO_LOW" + } + ], + "baselineEstimatedTokens": 21780, + "maximumPackEstimatedTokens": 1420, + "minimumEstimatedReduction": 0.934803, + "tokenEstimator": "portable ceil(characters / 4)", + "providerBillingMeasurement": false, + "cases": [ + { + "fileIndex": 0, + "path": "src/domain-0/module-000.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5608 + }, + { + "fileIndex": 1, + "path": "src/domain-1/module-001.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5608 + }, + { + "fileIndex": 2, + "path": "src/domain-2/module-002.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5608 + }, + { + "fileIndex": 3, + "path": "src/domain-3/module-003.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5607 + }, + { + "fileIndex": 4, + "path": "docs/domain-4/runbook-004.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5612 + }, + { + "fileIndex": 5, + "path": "src/domain-5/module-005.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5608 + }, + { + "fileIndex": 6, + "path": "src/domain-6/module-006.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5608 + }, + { + "fileIndex": 7, + "path": "src/domain-0/module-007.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5608 + }, + { + "fileIndex": 8, + "path": "src/domain-1/module-008.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5607 + }, + { + "fileIndex": 9, + "path": "docs/domain-2/runbook-009.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5607 + }, + { + "fileIndex": 10, + "path": "src/domain-3/module-010.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5609 + }, + { + "fileIndex": 11, + "path": "src/domain-4/module-011.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5609 + }, + { + "fileIndex": 12, + "path": "src/domain-5/module-012.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5609 + }, + { + "fileIndex": 13, + "path": "src/domain-6/module-013.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5609 + }, + { + "fileIndex": 14, + "path": "docs/domain-0/runbook-014.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1404, + "usedChars": 5613 + }, + { + "fileIndex": 15, + "path": "src/domain-1/module-015.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5609 + }, + { + "fileIndex": 16, + "path": "src/domain-2/module-016.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5609 + }, + { + "fileIndex": 17, + "path": "src/domain-3/module-017.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5609 + }, + { + "fileIndex": 18, + "path": "src/domain-4/module-018.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5608 + }, + { + "fileIndex": 19, + "path": "docs/domain-5/runbook-019.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1404, + "usedChars": 5613 + }, + { + "fileIndex": 20, + "path": "src/domain-6/module-020.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5609 + }, + { + "fileIndex": 21, + "path": "src/domain-0/module-021.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5609 + }, + { + "fileIndex": 22, + "path": "src/domain-1/module-022.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5609 + }, + { + "fileIndex": 23, + "path": "src/domain-2/module-023.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5609 + }, + { + "fileIndex": 24, + "path": "docs/domain-3/runbook-024.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1404, + "usedChars": 5613 + }, + { + "fileIndex": 25, + "path": "src/domain-4/module-025.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5609 + }, + { + "fileIndex": 26, + "path": "src/domain-5/module-026.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5609 + }, + { + "fileIndex": 27, + "path": "src/domain-6/module-027.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5609 + }, + { + "fileIndex": 28, + "path": "src/domain-0/module-028.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5608 + }, + { + "fileIndex": 29, + "path": "docs/domain-1/runbook-029.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1404, + "usedChars": 5613 + }, + { + "fileIndex": 30, + "path": "src/domain-2/module-030.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5608 + }, + { + "fileIndex": 31, + "path": "src/domain-3/module-031.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5608 + }, + { + "fileIndex": 32, + "path": "src/domain-4/module-032.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5608 + }, + { + "fileIndex": 33, + "path": "src/domain-5/module-033.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5608 + }, + { + "fileIndex": 34, + "path": "docs/domain-6/runbook-034.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5612 + }, + { + "fileIndex": 35, + "path": "src/domain-0/module-035.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5608 + }, + { + "fileIndex": 36, + "path": "src/domain-1/module-036.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5608 + }, + { + "fileIndex": 37, + "path": "src/domain-2/module-037.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5608 + }, + { + "fileIndex": 38, + "path": "src/domain-3/module-038.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5608 + }, + { + "fileIndex": 39, + "path": "docs/domain-4/runbook-039.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5607 + }, + { + "fileIndex": 0, + "path": "src/domain-0/module-000.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5675 + }, + { + "fileIndex": 1, + "path": "src/domain-1/module-001.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5674 + }, + { + "fileIndex": 2, + "path": "src/domain-2/module-002.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5674 + }, + { + "fileIndex": 3, + "path": "src/domain-3/module-003.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5674 + }, + { + "fileIndex": 4, + "path": "docs/domain-4/runbook-004.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 5, + "path": "src/domain-5/module-005.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5674 + }, + { + "fileIndex": 6, + "path": "src/domain-6/module-006.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5674 + }, + { + "fileIndex": 7, + "path": "src/domain-0/module-007.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5674 + }, + { + "fileIndex": 8, + "path": "src/domain-1/module-008.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5674 + }, + { + "fileIndex": 9, + "path": "docs/domain-2/runbook-009.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 10, + "path": "src/domain-3/module-010.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5675 + }, + { + "fileIndex": 11, + "path": "src/domain-4/module-011.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5675 + }, + { + "fileIndex": 12, + "path": "src/domain-5/module-012.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 13, + "path": "src/domain-6/module-013.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 14, + "path": "docs/domain-0/runbook-014.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1420, + "usedChars": 5680 + }, + { + "fileIndex": 15, + "path": "src/domain-1/module-015.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 16, + "path": "src/domain-2/module-016.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 17, + "path": "src/domain-3/module-017.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 18, + "path": "src/domain-4/module-018.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 19, + "path": "docs/domain-5/runbook-019.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1420, + "usedChars": 5680 + }, + { + "fileIndex": 20, + "path": "src/domain-6/module-020.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5675 + }, + { + "fileIndex": 21, + "path": "src/domain-0/module-021.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 22, + "path": "src/domain-1/module-022.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5675 + }, + { + "fileIndex": 23, + "path": "src/domain-2/module-023.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 24, + "path": "docs/domain-3/runbook-024.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1420, + "usedChars": 5680 + }, + { + "fileIndex": 25, + "path": "src/domain-4/module-025.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 26, + "path": "src/domain-5/module-026.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 27, + "path": "src/domain-6/module-027.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 28, + "path": "src/domain-0/module-028.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 29, + "path": "docs/domain-1/runbook-029.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1420, + "usedChars": 5680 + }, + { + "fileIndex": 30, + "path": "src/domain-2/module-030.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5675 + }, + { + "fileIndex": 31, + "path": "src/domain-3/module-031.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 32, + "path": "src/domain-4/module-032.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 33, + "path": "src/domain-5/module-033.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5675 + }, + { + "fileIndex": 34, + "path": "docs/domain-6/runbook-034.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1420, + "usedChars": 5680 + }, + { + "fileIndex": 35, + "path": "src/domain-0/module-035.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 36, + "path": "src/domain-1/module-036.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 37, + "path": "src/domain-2/module-037.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 38, + "path": "src/domain-3/module-038.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 39, + "path": "docs/domain-4/runbook-039.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1420, + "usedChars": 5680 + } + ], + "parity": [ + { + "fileIndex": 0, + "persistedRank": 1, + "inMemoryRank": 1, + "sameTargetRank": true + }, + { + "fileIndex": 20, + "persistedRank": 1, + "inMemoryRank": 1, + "sameTargetRank": true + }, + { + "fileIndex": 39, + "persistedRank": 1, + "inMemoryRank": 1, + "sameTargetRank": true + } + ], + "storeVerified": true + }, + { + "fileCount": 50, + "eventCount": 55, + "exactQueries": 50, + "typoTolerantQueries": 50, + "unsupportedQueries": 3, + "allTargetsRankedFirst": true, + "allAnswersPreserved": true, + "allExactQueriesUsedSqlite": true, + "allTypoQueriesUsedFuzzyRetrieval": true, + "allExactQueriesAcceptedDirect": true, + "allTypoQueriesConservativelyAbstained": true, + "persistedInMemoryParity": true, + "projectStructure": { + "filesScanned": 50, + "graphFileNodes": 50, + "graphReferenceEdges": 50, + "latePathAndReferencePreserved": true, + "markdownFirstMiddleLastPathsPreserved": true + }, + "graphLinkedDecisionRecovered": true, + "supersededDecisionExcluded": true, + "conflictVisible": true, + "sqliteReadModel": { + "schemaVersion": 1, + "eventCountMatchesLedger": true, + "headMatchesLedger": true, + "requiredTablesPresent": true, + "lateFileEvidenceFound": true + }, + "derivedRepair": { + "staleGraphDetectedAndRebuilt": true, + "staleMarkdownDetectedAndRebuilt": true, + "repairedRetrievalRank": 1 + }, + "correctAbstention": true, + "abstention": [ + { + "query": "unsupported50 nebula authorization secret", + "correctAbstention": true, + "errorCode": "CONTEXT_COVERAGE_TOO_LOW" + }, + { + "query": "missing50 lunar rollback credential", + "correctAbstention": true, + "errorCode": "CONTEXT_COVERAGE_TOO_LOW" + }, + { + "query": "absent50 oceanic encryption answer", + "correctAbstention": true, + "errorCode": "CONTEXT_COVERAGE_TOO_LOW" + } + ], + "baselineEstimatedTokens": 26785, + "maximumPackEstimatedTokens": 1421, + "minimumEstimatedReduction": 0.946948, + "tokenEstimator": "portable ceil(characters / 4)", + "providerBillingMeasurement": false, + "cases": [ + { + "fileIndex": 0, + "path": "src/domain-0/module-000.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5608 + }, + { + "fileIndex": 1, + "path": "src/domain-1/module-001.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5608 + }, + { + "fileIndex": 2, + "path": "src/domain-2/module-002.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5608 + }, + { + "fileIndex": 3, + "path": "src/domain-3/module-003.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5608 + }, + { + "fileIndex": 4, + "path": "docs/domain-4/runbook-004.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5611 + }, + { + "fileIndex": 5, + "path": "src/domain-5/module-005.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5608 + }, + { + "fileIndex": 6, + "path": "src/domain-6/module-006.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5608 + }, + { + "fileIndex": 7, + "path": "src/domain-0/module-007.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5608 + }, + { + "fileIndex": 8, + "path": "src/domain-1/module-008.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5607 + }, + { + "fileIndex": 9, + "path": "docs/domain-2/runbook-009.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5607 + }, + { + "fileIndex": 10, + "path": "src/domain-3/module-010.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5609 + }, + { + "fileIndex": 11, + "path": "src/domain-4/module-011.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5609 + }, + { + "fileIndex": 12, + "path": "src/domain-5/module-012.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5609 + }, + { + "fileIndex": 13, + "path": "src/domain-6/module-013.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5609 + }, + { + "fileIndex": 14, + "path": "docs/domain-0/runbook-014.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1404, + "usedChars": 5613 + }, + { + "fileIndex": 15, + "path": "src/domain-1/module-015.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5609 + }, + { + "fileIndex": 16, + "path": "src/domain-2/module-016.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5609 + }, + { + "fileIndex": 17, + "path": "src/domain-3/module-017.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5609 + }, + { + "fileIndex": 18, + "path": "src/domain-4/module-018.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5608 + }, + { + "fileIndex": 19, + "path": "docs/domain-5/runbook-019.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1404, + "usedChars": 5613 + }, + { + "fileIndex": 20, + "path": "src/domain-6/module-020.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5609 + }, + { + "fileIndex": 21, + "path": "src/domain-0/module-021.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5609 + }, + { + "fileIndex": 22, + "path": "src/domain-1/module-022.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5609 + }, + { + "fileIndex": 23, + "path": "src/domain-2/module-023.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5609 + }, + { + "fileIndex": 24, + "path": "docs/domain-3/runbook-024.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1404, + "usedChars": 5613 + }, + { + "fileIndex": 25, + "path": "src/domain-4/module-025.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5609 + }, + { + "fileIndex": 26, + "path": "src/domain-5/module-026.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5609 + }, + { + "fileIndex": 27, + "path": "src/domain-6/module-027.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5609 + }, + { + "fileIndex": 28, + "path": "src/domain-0/module-028.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5608 + }, + { + "fileIndex": 29, + "path": "docs/domain-1/runbook-029.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1404, + "usedChars": 5613 + }, + { + "fileIndex": 30, + "path": "src/domain-2/module-030.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5609 + }, + { + "fileIndex": 31, + "path": "src/domain-3/module-031.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5609 + }, + { + "fileIndex": 32, + "path": "src/domain-4/module-032.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5609 + }, + { + "fileIndex": 33, + "path": "src/domain-5/module-033.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5609 + }, + { + "fileIndex": 34, + "path": "docs/domain-6/runbook-034.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1404, + "usedChars": 5613 + }, + { + "fileIndex": 35, + "path": "src/domain-0/module-035.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5609 + }, + { + "fileIndex": 36, + "path": "src/domain-1/module-036.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5609 + }, + { + "fileIndex": 37, + "path": "src/domain-2/module-037.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5609 + }, + { + "fileIndex": 38, + "path": "src/domain-3/module-038.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5608 + }, + { + "fileIndex": 39, + "path": "docs/domain-4/runbook-039.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1404, + "usedChars": 5613 + }, + { + "fileIndex": 40, + "path": "src/domain-5/module-040.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5608 + }, + { + "fileIndex": 41, + "path": "src/domain-6/module-041.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5608 + }, + { + "fileIndex": 42, + "path": "src/domain-0/module-042.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5608 + }, + { + "fileIndex": 43, + "path": "src/domain-1/module-043.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5608 + }, + { + "fileIndex": 44, + "path": "docs/domain-2/runbook-044.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1403, + "usedChars": 5612 + }, + { + "fileIndex": 45, + "path": "src/domain-3/module-045.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5608 + }, + { + "fileIndex": 46, + "path": "src/domain-4/module-046.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5608 + }, + { + "fileIndex": 47, + "path": "src/domain-5/module-047.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5608 + }, + { + "fileIndex": 48, + "path": "src/domain-6/module-048.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5608 + }, + { + "fileIndex": 49, + "path": "docs/domain-0/runbook-049.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1402, + "usedChars": 5607 + }, + { + "fileIndex": 0, + "path": "src/domain-0/module-000.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5675 + }, + { + "fileIndex": 1, + "path": "src/domain-1/module-001.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5674 + }, + { + "fileIndex": 2, + "path": "src/domain-2/module-002.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5674 + }, + { + "fileIndex": 3, + "path": "src/domain-3/module-003.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5674 + }, + { + "fileIndex": 4, + "path": "docs/domain-4/runbook-004.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1421, + "usedChars": 5682 + }, + { + "fileIndex": 5, + "path": "src/domain-5/module-005.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5674 + }, + { + "fileIndex": 6, + "path": "src/domain-6/module-006.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5674 + }, + { + "fileIndex": 7, + "path": "src/domain-0/module-007.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5674 + }, + { + "fileIndex": 8, + "path": "src/domain-1/module-008.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5674 + }, + { + "fileIndex": 9, + "path": "docs/domain-2/runbook-009.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 10, + "path": "src/domain-3/module-010.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5675 + }, + { + "fileIndex": 11, + "path": "src/domain-4/module-011.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5675 + }, + { + "fileIndex": 12, + "path": "src/domain-5/module-012.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 13, + "path": "src/domain-6/module-013.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 14, + "path": "docs/domain-0/runbook-014.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1420, + "usedChars": 5680 + }, + { + "fileIndex": 15, + "path": "src/domain-1/module-015.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 16, + "path": "src/domain-2/module-016.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 17, + "path": "src/domain-3/module-017.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 18, + "path": "src/domain-4/module-018.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 19, + "path": "docs/domain-5/runbook-019.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1420, + "usedChars": 5680 + }, + { + "fileIndex": 20, + "path": "src/domain-6/module-020.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5675 + }, + { + "fileIndex": 21, + "path": "src/domain-0/module-021.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 22, + "path": "src/domain-1/module-022.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5675 + }, + { + "fileIndex": 23, + "path": "src/domain-2/module-023.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 24, + "path": "docs/domain-3/runbook-024.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1420, + "usedChars": 5680 + }, + { + "fileIndex": 25, + "path": "src/domain-4/module-025.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 26, + "path": "src/domain-5/module-026.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 27, + "path": "src/domain-6/module-027.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 28, + "path": "src/domain-0/module-028.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 29, + "path": "docs/domain-1/runbook-029.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1420, + "usedChars": 5680 + }, + { + "fileIndex": 30, + "path": "src/domain-2/module-030.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5675 + }, + { + "fileIndex": 31, + "path": "src/domain-3/module-031.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 32, + "path": "src/domain-4/module-032.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 33, + "path": "src/domain-5/module-033.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5675 + }, + { + "fileIndex": 34, + "path": "docs/domain-6/runbook-034.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1420, + "usedChars": 5680 + }, + { + "fileIndex": 35, + "path": "src/domain-0/module-035.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 36, + "path": "src/domain-1/module-036.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 37, + "path": "src/domain-2/module-037.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 38, + "path": "src/domain-3/module-038.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 39, + "path": "docs/domain-4/runbook-039.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1420, + "usedChars": 5680 + }, + { + "fileIndex": 40, + "path": "src/domain-5/module-040.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5675 + }, + { + "fileIndex": 41, + "path": "src/domain-6/module-041.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 42, + "path": "src/domain-0/module-042.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 43, + "path": "src/domain-1/module-043.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 44, + "path": "docs/domain-2/runbook-044.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1421, + "usedChars": 5683 + }, + { + "fileIndex": 45, + "path": "src/domain-3/module-045.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 46, + "path": "src/domain-4/module-046.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 47, + "path": "src/domain-5/module-047.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 48, + "path": "src/domain-6/module-048.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1419, + "usedChars": 5676 + }, + { + "fileIndex": 49, + "path": "docs/domain-0/runbook-049.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1420, + "usedChars": 5680 + } + ], + "parity": [ + { + "fileIndex": 0, + "persistedRank": 1, + "inMemoryRank": 1, + "sameTargetRank": true + }, + { + "fileIndex": 25, + "persistedRank": 1, + "inMemoryRank": 1, + "sameTargetRank": true + }, + { + "fileIndex": 49, + "persistedRank": 1, + "inMemoryRank": 1, + "sameTargetRank": true + } + ], + "storeVerified": true + }, + { + "fileCount": 100, + "eventCount": 105, + "exactQueries": 100, + "typoTolerantQueries": 100, + "unsupportedQueries": 3, + "allTargetsRankedFirst": true, + "allAnswersPreserved": true, + "allExactQueriesUsedSqlite": true, + "allTypoQueriesUsedFuzzyRetrieval": true, + "allExactQueriesAcceptedDirect": true, + "allTypoQueriesConservativelyAbstained": true, + "persistedInMemoryParity": true, + "projectStructure": { + "filesScanned": 100, + "graphFileNodes": 100, + "graphReferenceEdges": 100, + "latePathAndReferencePreserved": true, + "markdownFirstMiddleLastPathsPreserved": true + }, + "graphLinkedDecisionRecovered": true, + "supersededDecisionExcluded": true, + "conflictVisible": true, + "sqliteReadModel": { + "schemaVersion": 1, + "eventCountMatchesLedger": true, + "headMatchesLedger": true, + "requiredTablesPresent": true, + "lateFileEvidenceFound": true + }, + "derivedRepair": { + "staleGraphDetectedAndRebuilt": true, + "staleMarkdownDetectedAndRebuilt": true, + "repairedRetrievalRank": 1 + }, + "correctAbstention": true, + "abstention": [ + { + "query": "unsupported100 nebula authorization secret", + "correctAbstention": true, + "errorCode": "CONTEXT_COVERAGE_TOO_LOW" + }, + { + "query": "missing100 lunar rollback credential", + "correctAbstention": true, + "errorCode": "CONTEXT_COVERAGE_TOO_LOW" + }, + { + "query": "absent100 oceanic encryption answer", + "correctAbstention": true, + "errorCode": "CONTEXT_COVERAGE_TOO_LOW" + } + ], + "baselineEstimatedTokens": 51897, + "maximumPackEstimatedTokens": 1478, + "minimumEstimatedReduction": 0.971521, + "tokenEstimator": "portable ceil(characters / 4)", + "providerBillingMeasurement": false, + "cases": [ + { + "fileIndex": 0, + "path": "src/domain-0/module-000.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 1, + "path": "src/domain-1/module-001.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 2, + "path": "src/domain-2/module-002.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 3, + "path": "src/domain-3/module-003.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 4, + "path": "docs/domain-4/runbook-004.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1460, + "usedChars": 5839 + }, + { + "fileIndex": 5, + "path": "src/domain-5/module-005.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 6, + "path": "src/domain-6/module-006.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 7, + "path": "src/domain-0/module-007.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 8, + "path": "src/domain-1/module-008.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5834 + }, + { + "fileIndex": 9, + "path": "docs/domain-2/runbook-009.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5833 + }, + { + "fileIndex": 10, + "path": "src/domain-3/module-010.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 11, + "path": "src/domain-4/module-011.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 12, + "path": "src/domain-5/module-012.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 13, + "path": "src/domain-6/module-013.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 14, + "path": "docs/domain-0/runbook-014.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1460, + "usedChars": 5839 + }, + { + "fileIndex": 15, + "path": "src/domain-1/module-015.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 16, + "path": "src/domain-2/module-016.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 17, + "path": "src/domain-3/module-017.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 18, + "path": "src/domain-4/module-018.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5834 + }, + { + "fileIndex": 19, + "path": "docs/domain-5/runbook-019.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1460, + "usedChars": 5839 + }, + { + "fileIndex": 20, + "path": "src/domain-6/module-020.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 21, + "path": "src/domain-0/module-021.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 22, + "path": "src/domain-1/module-022.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 23, + "path": "src/domain-2/module-023.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 24, + "path": "docs/domain-3/runbook-024.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1460, + "usedChars": 5839 + }, + { + "fileIndex": 25, + "path": "src/domain-4/module-025.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 26, + "path": "src/domain-5/module-026.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 27, + "path": "src/domain-6/module-027.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 28, + "path": "src/domain-0/module-028.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5834 + }, + { + "fileIndex": 29, + "path": "docs/domain-1/runbook-029.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1460, + "usedChars": 5839 + }, + { + "fileIndex": 30, + "path": "src/domain-2/module-030.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 31, + "path": "src/domain-3/module-031.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 32, + "path": "src/domain-4/module-032.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 33, + "path": "src/domain-5/module-033.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 34, + "path": "docs/domain-6/runbook-034.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1460, + "usedChars": 5839 + }, + { + "fileIndex": 35, + "path": "src/domain-0/module-035.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 36, + "path": "src/domain-1/module-036.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 37, + "path": "src/domain-2/module-037.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 38, + "path": "src/domain-3/module-038.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5834 + }, + { + "fileIndex": 39, + "path": "docs/domain-4/runbook-039.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1460, + "usedChars": 5839 + }, + { + "fileIndex": 40, + "path": "src/domain-5/module-040.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 41, + "path": "src/domain-6/module-041.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 42, + "path": "src/domain-0/module-042.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 43, + "path": "src/domain-1/module-043.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 44, + "path": "docs/domain-2/runbook-044.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1460, + "usedChars": 5839 + }, + { + "fileIndex": 45, + "path": "src/domain-3/module-045.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 46, + "path": "src/domain-4/module-046.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 47, + "path": "src/domain-5/module-047.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 48, + "path": "src/domain-6/module-048.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5834 + }, + { + "fileIndex": 49, + "path": "docs/domain-0/runbook-049.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1460, + "usedChars": 5839 + }, + { + "fileIndex": 50, + "path": "src/domain-1/module-050.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 51, + "path": "src/domain-2/module-051.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 52, + "path": "src/domain-3/module-052.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 53, + "path": "src/domain-4/module-053.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 54, + "path": "docs/domain-5/runbook-054.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1460, + "usedChars": 5839 + }, + { + "fileIndex": 55, + "path": "src/domain-6/module-055.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 56, + "path": "src/domain-0/module-056.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 57, + "path": "src/domain-1/module-057.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 58, + "path": "src/domain-2/module-058.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5834 + }, + { + "fileIndex": 59, + "path": "docs/domain-3/runbook-059.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1460, + "usedChars": 5839 + }, + { + "fileIndex": 60, + "path": "src/domain-4/module-060.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 61, + "path": "src/domain-5/module-061.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 62, + "path": "src/domain-6/module-062.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 63, + "path": "src/domain-0/module-063.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 64, + "path": "docs/domain-1/runbook-064.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1460, + "usedChars": 5839 + }, + { + "fileIndex": 65, + "path": "src/domain-2/module-065.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 66, + "path": "src/domain-3/module-066.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 67, + "path": "src/domain-4/module-067.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 68, + "path": "src/domain-5/module-068.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5834 + }, + { + "fileIndex": 69, + "path": "docs/domain-6/runbook-069.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1460, + "usedChars": 5839 + }, + { + "fileIndex": 70, + "path": "src/domain-0/module-070.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 71, + "path": "src/domain-1/module-071.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 72, + "path": "src/domain-2/module-072.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 73, + "path": "src/domain-3/module-073.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 74, + "path": "docs/domain-4/runbook-074.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1460, + "usedChars": 5839 + }, + { + "fileIndex": 75, + "path": "src/domain-5/module-075.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 76, + "path": "src/domain-6/module-076.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 77, + "path": "src/domain-0/module-077.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 78, + "path": "src/domain-1/module-078.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5834 + }, + { + "fileIndex": 79, + "path": "docs/domain-2/runbook-079.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1460, + "usedChars": 5839 + }, + { + "fileIndex": 80, + "path": "src/domain-3/module-080.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 81, + "path": "src/domain-4/module-081.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 82, + "path": "src/domain-5/module-082.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 83, + "path": "src/domain-6/module-083.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 84, + "path": "docs/domain-0/runbook-084.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1460, + "usedChars": 5839 + }, + { + "fileIndex": 85, + "path": "src/domain-1/module-085.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 86, + "path": "src/domain-2/module-086.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 87, + "path": "src/domain-3/module-087.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 88, + "path": "src/domain-4/module-088.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5834 + }, + { + "fileIndex": 89, + "path": "docs/domain-5/runbook-089.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1460, + "usedChars": 5839 + }, + { + "fileIndex": 90, + "path": "src/domain-6/module-090.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5835 + }, + { + "fileIndex": 91, + "path": "src/domain-0/module-091.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5834 + }, + { + "fileIndex": 92, + "path": "src/domain-1/module-092.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5834 + }, + { + "fileIndex": 93, + "path": "src/domain-2/module-093.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5834 + }, + { + "fileIndex": 94, + "path": "docs/domain-3/runbook-094.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1460, + "usedChars": 5838 + }, + { + "fileIndex": 95, + "path": "src/domain-4/module-095.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5834 + }, + { + "fileIndex": 96, + "path": "src/domain-5/module-096.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5834 + }, + { + "fileIndex": 97, + "path": "src/domain-6/module-097.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5834 + }, + { + "fileIndex": 98, + "path": "src/domain-0/module-098.js", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1459, + "usedChars": 5834 + }, + { + "fileIndex": 99, + "path": "docs/domain-1/runbook-099.md", + "queryType": "exact", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": true, + "fuzzyCandidateUsed": true, + "coverage": "direct", + "evidenceDecision": "ACCEPT_DIRECT", + "usedTokens": 1458, + "usedChars": 5832 + }, + { + "fileIndex": 0, + "path": "src/domain-0/module-000.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 1, + "path": "src/domain-1/module-001.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 2, + "path": "src/domain-2/module-002.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 3, + "path": "src/domain-3/module-003.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 4, + "path": "docs/domain-4/runbook-004.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1478, + "usedChars": 5909 + }, + { + "fileIndex": 5, + "path": "src/domain-5/module-005.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 6, + "path": "src/domain-6/module-006.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 7, + "path": "src/domain-0/module-007.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 8, + "path": "src/domain-1/module-008.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 9, + "path": "docs/domain-2/runbook-009.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1478, + "usedChars": 5909 + }, + { + "fileIndex": 10, + "path": "src/domain-3/module-010.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 11, + "path": "src/domain-4/module-011.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 12, + "path": "src/domain-5/module-012.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 13, + "path": "src/domain-6/module-013.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 14, + "path": "docs/domain-0/runbook-014.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1477, + "usedChars": 5905 + }, + { + "fileIndex": 15, + "path": "src/domain-1/module-015.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 16, + "path": "src/domain-2/module-016.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 17, + "path": "src/domain-3/module-017.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 18, + "path": "src/domain-4/module-018.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 19, + "path": "docs/domain-5/runbook-019.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1477, + "usedChars": 5905 + }, + { + "fileIndex": 20, + "path": "src/domain-6/module-020.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 21, + "path": "src/domain-0/module-021.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 22, + "path": "src/domain-1/module-022.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 23, + "path": "src/domain-2/module-023.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 24, + "path": "docs/domain-3/runbook-024.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1477, + "usedChars": 5905 + }, + { + "fileIndex": 25, + "path": "src/domain-4/module-025.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 26, + "path": "src/domain-5/module-026.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 27, + "path": "src/domain-6/module-027.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 28, + "path": "src/domain-0/module-028.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 29, + "path": "docs/domain-1/runbook-029.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1477, + "usedChars": 5905 + }, + { + "fileIndex": 30, + "path": "src/domain-2/module-030.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 31, + "path": "src/domain-3/module-031.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 32, + "path": "src/domain-4/module-032.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 33, + "path": "src/domain-5/module-033.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 34, + "path": "docs/domain-6/runbook-034.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1477, + "usedChars": 5905 + }, + { + "fileIndex": 35, + "path": "src/domain-0/module-035.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 36, + "path": "src/domain-1/module-036.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 37, + "path": "src/domain-2/module-037.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 38, + "path": "src/domain-3/module-038.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 39, + "path": "docs/domain-4/runbook-039.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1477, + "usedChars": 5905 + }, + { + "fileIndex": 40, + "path": "src/domain-5/module-040.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 41, + "path": "src/domain-6/module-041.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 42, + "path": "src/domain-0/module-042.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 43, + "path": "src/domain-1/module-043.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 44, + "path": "docs/domain-2/runbook-044.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1478, + "usedChars": 5909 + }, + { + "fileIndex": 45, + "path": "src/domain-3/module-045.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 46, + "path": "src/domain-4/module-046.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 47, + "path": "src/domain-5/module-047.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 48, + "path": "src/domain-6/module-048.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 49, + "path": "docs/domain-0/runbook-049.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1477, + "usedChars": 5906 + }, + { + "fileIndex": 50, + "path": "src/domain-1/module-050.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 51, + "path": "src/domain-2/module-051.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 52, + "path": "src/domain-3/module-052.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 53, + "path": "src/domain-4/module-053.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 54, + "path": "docs/domain-5/runbook-054.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1477, + "usedChars": 5905 + }, + { + "fileIndex": 55, + "path": "src/domain-6/module-055.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 56, + "path": "src/domain-0/module-056.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 57, + "path": "src/domain-1/module-057.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 58, + "path": "src/domain-2/module-058.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 59, + "path": "docs/domain-3/runbook-059.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1477, + "usedChars": 5905 + }, + { + "fileIndex": 60, + "path": "src/domain-4/module-060.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 61, + "path": "src/domain-5/module-061.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 62, + "path": "src/domain-6/module-062.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 63, + "path": "src/domain-0/module-063.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 64, + "path": "docs/domain-1/runbook-064.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1477, + "usedChars": 5905 + }, + { + "fileIndex": 65, + "path": "src/domain-2/module-065.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 66, + "path": "src/domain-3/module-066.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 67, + "path": "src/domain-4/module-067.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 68, + "path": "src/domain-5/module-068.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 69, + "path": "docs/domain-6/runbook-069.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1477, + "usedChars": 5905 + }, + { + "fileIndex": 70, + "path": "src/domain-0/module-070.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 71, + "path": "src/domain-1/module-071.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 72, + "path": "src/domain-2/module-072.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 73, + "path": "src/domain-3/module-073.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 74, + "path": "docs/domain-4/runbook-074.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1477, + "usedChars": 5905 + }, + { + "fileIndex": 75, + "path": "src/domain-5/module-075.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 76, + "path": "src/domain-6/module-076.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 77, + "path": "src/domain-0/module-077.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 78, + "path": "src/domain-1/module-078.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 79, + "path": "docs/domain-2/runbook-079.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1477, + "usedChars": 5905 + }, + { + "fileIndex": 80, + "path": "src/domain-3/module-080.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 81, + "path": "src/domain-4/module-081.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 82, + "path": "src/domain-5/module-082.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 83, + "path": "src/domain-6/module-083.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 84, + "path": "docs/domain-0/runbook-084.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1477, + "usedChars": 5905 + }, + { + "fileIndex": 85, + "path": "src/domain-1/module-085.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 86, + "path": "src/domain-2/module-086.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 87, + "path": "src/domain-3/module-087.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1475, + "usedChars": 5900 + }, + { + "fileIndex": 88, + "path": "src/domain-4/module-088.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 89, + "path": "docs/domain-5/runbook-089.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1477, + "usedChars": 5905 + }, + { + "fileIndex": 90, + "path": "src/domain-6/module-090.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 91, + "path": "src/domain-0/module-091.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 92, + "path": "src/domain-1/module-092.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 93, + "path": "src/domain-2/module-093.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 94, + "path": "docs/domain-3/runbook-094.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1477, + "usedChars": 5906 + }, + { + "fileIndex": 95, + "path": "src/domain-4/module-095.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 96, + "path": "src/domain-5/module-096.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 97, + "path": "src/domain-6/module-097.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 98, + "path": "src/domain-0/module-098.js", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1476, + "usedChars": 5901 + }, + { + "fileIndex": 99, + "path": "docs/domain-1/runbook-099.md", + "queryType": "typo-tolerant", + "targetRank": 1, + "answerPreserved": true, + "citationHashPresent": true, + "manifestHashPresent": true, + "sqliteCandidateUsed": false, + "fuzzyCandidateUsed": true, + "coverage": "partial", + "evidenceDecision": "ABSTAIN", + "usedTokens": 1478, + "usedChars": 5909 + } + ], + "parity": [ + { + "fileIndex": 0, + "persistedRank": 1, + "inMemoryRank": 1, + "sameTargetRank": true + }, + { + "fileIndex": 50, + "persistedRank": 1, + "inMemoryRank": 1, + "sameTargetRank": true + }, + { + "fileIndex": 99, + "persistedRank": 1, + "inMemoryRank": 1, + "sameTargetRank": true + } + ], + "storeVerified": true + } + ] + }, + "claim": "Across deterministic 40-, 50-, and 100-file workspaces, every file-specific exact and typo-tolerant query ranked its cited answer first; SQLite, graph, Markdown, supersession, conflict, repair, and unsupported-query controls also passed.", + "limitations": [ + "This is a deterministic local retrieval and projection-integrity benchmark, not a provider-backed task-completion experiment.", + "The files and answer records are synthetic so every relevance judgment and expected answer is reproducible.", + "Correct abstention means unsupported direct-coverage queries were rejected; it is successful fail-closed behavior, not a long-document failure.", + "Portable ceil(characters / 4) estimates are not provider billing measurements." + ] +} diff --git a/bench/results/research-retrieval-development-v0.4.json b/bench/results/research-retrieval-development-v0.4.json new file mode 100644 index 0000000..98b086a --- /dev/null +++ b/bench/results/research-retrieval-development-v0.4.json @@ -0,0 +1,192009 @@ +{ + "schemaVersion": "qarinah.research-retrieval-development-result.v4", + "packageVersion": "0.1.6", + "status": "current-production-recomputation-on-inspected-development-corpus", + "confirmatoryClaimEligible": false, + "implementation": { + "algorithm": "sha256-path-lf-content-v1", + "productVersion": "0.1.6", + "rankingProfile": "admission-first-v2", + "evidenceSufficiencyMethod": "evidence-sufficiency-v2", + "files": [ + { + "path": "bench/research/swe-bench-lite.mjs", + "sha256": "sha256:3b92352951a07854786b1a74ee5d2e6e5cbe1247b7c39d2f1135593cfed431dc" + }, + { + "path": "scripts/evaluate-research-retrieval-v0.2.mjs", + "sha256": "sha256:744b0361367558f7cd43756e9267d8139692fc6bed06599b83d5a53aa915c91f" + }, + { + "path": "scripts/evaluate-research-retrieval-v0.4.mjs", + "sha256": "sha256:3484968eae2922e194e089b9a650101e9717fcf274bdc4a2e63a3744064f4e8d" + }, + { + "path": "src/canonical.js", + "sha256": "sha256:c24859c69ff8571128107c7de6718fc02aad9cb64f807f174d23bf8b12293225" + }, + { + "path": "src/contracts.js", + "sha256": "sha256:d74d0487fad186901c7aa1a8c8530c0920fe3908c611ce85ec17c6336d575650" + }, + { + "path": "src/index.js", + "sha256": "sha256:88a9226efbf1f499209de3e71df4e28eb326ef47c12658b971f180e94b8f586a" + }, + { + "path": "src/indexer.js", + "sha256": "sha256:868c6e433dc858cd665c3c844bb72449e102bf1bc288f1c9daf41ecf4986ff4b" + }, + { + "path": "src/interoperability/boundary.js", + "sha256": "sha256:80798113257019fa38573acf262ed69b8f1b2b887ceb8ce37f53951c2f1d3118" + }, + { + "path": "src/redact.js", + "sha256": "sha256:6198154b1d4a37adfea308f8b2723c89788ab8046ca587210e278952ca4454b4" + }, + { + "path": "src/retrieval.js", + "sha256": "sha256:4088ffb52c60be9dae0a12feaf4ebe158e12d81a3f8028b883987a5dd23c34e0" + } + ], + "digest": "sha256:034498c05fdf847078e639b6a2ed1efe2c72a1d714ca7f9bee337b3312bcd2eb" + }, + "historicalLineage": { + "v02": { + "tag": "research-retrieval-development-v0.2", + "commit": "bd566ac5ba7b302653b994fd0622d516fa74bbb8", + "artifact": "bench/results/research-retrieval-development-v0.2.json", + "artifactSha256": "sha256:bfe8015811ffbecd5e3c00eb9f4a1e104478605cd605442a1ec96d67582e4b3f" + }, + "v03": { + "artifact": "bench/results/research-sufficiency-development-v0.3.json", + "interpretation": "Historical post-hoc threshold calibration over the frozen evidence-sufficiency-v1 scores in v0.2; preserved unchanged." + }, + "current": "Production evidence-sufficiency-v2 scores and decisions recomputed as development v0.4." + }, + "executionScope": { + "providerModelCalls": 0, + "providerReportedTokens": false, + "sweBenchDockerTaskExecution": false, + "humanRelevanceReview": false, + "humanCodeReview": false, + "claimBoundary": "Development evidence only. The corpus was already inspected, the v0.4 run is bound to current production source hashes, and structural oracle labels are not human relevance judgments." + }, + "expected": { + "corpus": { + "digest": "sha256:01b35115ac639c1fcd3779561f83d5bb21988eb74ee5e93798c5d7579d757863", + "rawTestParquetSha256": "sha256:7a21f37b8bc179c7db5beeb14e88ac538ba283455c776e6b2535bbfb6e3551b4", + "officialPageDeclaredRepositories": 11, + "pinnedRevisionObservedRepositories": 12, + "discrepancyRecorded": true, + "exploratoryReuse": true, + "tasks": 300, + "heldoutTasks": 240 + }, + "settings": { + "static": { + "tasks": 240, + "positiveUnderStructuralOracle": 191, + "noPositiveUnderStructuralOracle": 49, + "methods": { + "bm25Admitted": { + "tasks": 240, + "scorableTasks": 191, + "meanRecallAt1": 0.222553, + "meanRecallAt5": 0.566714, + "meanRecallAt10": 0.762613, + "meanPrecisionAt5": 0.388482, + "meanPrecisionAt10": 0.285864, + "meanReciprocalRank": 0.70321, + "meanNdcgAt10": 0.643283, + "meanDirectRecallAt10": 0.860577, + "totalEstimatedContextTokens": 633849 + }, + "balancedV1": { + "tasks": 240, + "scorableTasks": 191, + "meanRecallAt1": 0.187609, + "meanRecallAt5": 0.538434, + "meanRecallAt10": 0.723703, + "meanPrecisionAt5": 0.398953, + "meanPrecisionAt10": 0.309424, + "meanReciprocalRank": 0.644255, + "meanNdcgAt10": 0.604213, + "meanDirectRecallAt10": 0.740385, + "totalEstimatedContextTokens": 588873 + }, + "qarinahV2": { + "tasks": 240, + "scorableTasks": 191, + "meanRecallAt1": 0.222553, + "meanRecallAt5": 0.566714, + "meanRecallAt10": 0.762613, + "meanPrecisionAt5": 0.388482, + "meanPrecisionAt10": 0.285864, + "meanReciprocalRank": 0.70321, + "meanNdcgAt10": 0.643283, + "meanDirectRecallAt10": 0.860577, + "totalEstimatedContextTokens": 634301 + }, + "qarinahV2NoGraph": { + "tasks": 240, + "scorableTasks": 191, + "meanRecallAt1": 0.222553, + "meanRecallAt5": 0.566714, + "meanRecallAt10": 0.762613, + "meanPrecisionAt5": 0.388482, + "meanPrecisionAt10": 0.285864, + "meanReciprocalRank": 0.70321, + "meanNdcgAt10": 0.643283, + "meanDirectRecallAt10": 0.860577, + "totalEstimatedContextTokens": 634301 + }, + "oracle": { + "tasks": 240, + "scorableTasks": 191, + "meanRecallAt1": 0.437609, + "meanRecallAt5": 0.874345, + "meanRecallAt10": 0.979058, + "meanPrecisionAt5": 0.591623, + "meanPrecisionAt10": 0.410995, + "meanReciprocalRank": 1, + "meanNdcgAt10": 1, + "meanDirectRecallAt10": 1, + "totalEstimatedContextTokens": 267269 + } + }, + "evidenceSufficiency": { + "positives": 191, + "noPositiveUnderStructuralOracle": 49, + "supported": 149, + "supportedPrecision": 0.818792, + "supportedRecall": 0.638743, + "noPositiveFalseAcceptanceRate": 0.55102, + "noPositiveCorrectAbstentionRate": 0.44898, + "rocAuc": 0.583876, + "prAucAveragePrecision": 0.8489, + "brierScore": 0.29705, + "expectedCalibrationError10Bin": 0.364664, + "riskCoverage": [ + { + "coverage": 0.1, + "threshold": 0.5875, + "risk": 0.083333 + }, + { + "coverage": 0.25, + "threshold": 0.502955, + "risk": 0.133333 + }, + { + "coverage": 0.5, + "threshold": 0.425893, + "risk": 0.166667 + }, + { + "coverage": 0.75, + "threshold": 0.356731, + "risk": 0.194444 + }, + { + "coverage": 1, + "threshold": 0.175403, + "risk": 0.204167 + } + ] + }, + "directDecision": { + "method": "evidence-sufficiency-v2", + "directThreshold": 0.65, + "partialThreshold": 0.4, + "tasks": 240, + "positives": 191, + "noPositiveUnderStructuralOracle": 49, + "acceptedDirect": 10, + "abstained": 230, + "truePositive": 10, + "falsePositive": 0, + "trueNegative": 49, + "falseNegative": 181, + "acceptedPrecision": 1, + "acceptedRecall": 0.052356, + "acceptedF1": 0.099502, + "falseAcceptanceRate": 0, + "correctAbstentionRate": 1, + "acceptanceCoverage": 0.041667, + "stateCounts": { + "DIRECTLY_SUPPORTED": 10, + "PARTIALLY_SUPPORTED": 139, + "INSUFFICIENT_EVIDENCE": 91 + }, + "confidenceIntervals95": { + "acceptedPrecision": { + "method": "Clopper-Pearson exact two-sided", + "successes": 10, + "trials": 10, + "lower": 0.691503, + "upper": 1 + }, + "falseAcceptanceRate": { + "method": "Clopper-Pearson exact two-sided", + "successes": 0, + "trials": 49, + "lower": 0, + "upper": 0.072519 + } + } + }, + "noTemporalAblation": { + "returnedItems": 2288, + "futureItems": 1083, + "futureItemRate": 0.473339, + "affectedQueries": 240, + "affectedQueryRate": 1 + }, + "budgetCurves": { + "512": { + "bm25Admitted": { + "meanRecallAt10": 0.286094, + "meanNdcgAt10": 0.342941, + "totalDeliveredTokens": 104908 + }, + "balancedV1": { + "meanRecallAt10": 0.268285, + "meanNdcgAt10": 0.329656, + "totalDeliveredTokens": 105715 + }, + "qarinahV2": { + "meanRecallAt10": 0.286094, + "meanNdcgAt10": 0.342941, + "totalDeliveredTokens": 105030 + }, + "qarinahV2NoGraph": { + "meanRecallAt10": 0.286094, + "meanNdcgAt10": 0.342941, + "totalDeliveredTokens": 105030 + }, + "oracle": { + "meanRecallAt10": 0.478066, + "meanNdcgAt10": 0.597489, + "totalDeliveredTokens": 67676 + } + }, + "1000": { + "bm25Admitted": { + "meanRecallAt10": 0.412423, + "meanNdcgAt10": 0.442285, + "totalDeliveredTokens": 207265 + }, + "balancedV1": { + "meanRecallAt10": 0.412105, + "meanNdcgAt10": 0.441669, + "totalDeliveredTokens": 210721 + }, + "qarinahV2": { + "meanRecallAt10": 0.412423, + "meanNdcgAt10": 0.442285, + "totalDeliveredTokens": 207265 + }, + "qarinahV2NoGraph": { + "meanRecallAt10": 0.412423, + "meanNdcgAt10": 0.442285, + "totalDeliveredTokens": 207265 + }, + "oracle": { + "meanRecallAt10": 0.73822, + "meanNdcgAt10": 0.826218, + "totalDeliveredTokens": 125392 + } + }, + "2000": { + "bm25Admitted": { + "meanRecallAt10": 0.576868, + "meanNdcgAt10": 0.542552, + "totalDeliveredTokens": 394583 + }, + "balancedV1": { + "meanRecallAt10": 0.560289, + "meanNdcgAt10": 0.515144, + "totalDeliveredTokens": 395710 + }, + "qarinahV2": { + "meanRecallAt10": 0.576868, + "meanNdcgAt10": 0.542552, + "totalDeliveredTokens": 394798 + }, + "qarinahV2NoGraph": { + "meanRecallAt10": 0.576868, + "meanNdcgAt10": 0.542552, + "totalDeliveredTokens": 394798 + }, + "oracle": { + "meanRecallAt10": 0.863279, + "meanNdcgAt10": 0.921855, + "totalDeliveredTokens": 199844 + } + }, + "4000": { + "bm25Admitted": { + "meanRecallAt10": 0.758171, + "meanNdcgAt10": 0.64065, + "totalDeliveredTokens": 676257 + }, + "balancedV1": { + "meanRecallAt10": 0.723703, + "meanNdcgAt10": 0.604213, + "totalDeliveredTokens": 685529 + }, + "qarinahV2": { + "meanRecallAt10": 0.758171, + "meanNdcgAt10": 0.64065, + "totalDeliveredTokens": 685781 + }, + "qarinahV2NoGraph": { + "meanRecallAt10": 0.758171, + "meanNdcgAt10": 0.64065, + "totalDeliveredTokens": 685781 + }, + "oracle": { + "meanRecallAt10": 0.979058, + "meanNdcgAt10": 1, + "totalDeliveredTokens": 273777 + } + }, + "8000": { + "bm25Admitted": { + "meanRecallAt10": 0.762613, + "meanNdcgAt10": 0.643283, + "totalDeliveredTokens": 1042637 + }, + "balancedV1": { + "meanRecallAt10": 0.723703, + "meanNdcgAt10": 0.604213, + "totalDeliveredTokens": 1052161 + }, + "qarinahV2": { + "meanRecallAt10": 0.762613, + "meanNdcgAt10": 0.643283, + "totalDeliveredTokens": 1052161 + }, + "qarinahV2NoGraph": { + "meanRecallAt10": 0.762613, + "meanNdcgAt10": 0.643283, + "totalDeliveredTokens": 1052161 + }, + "oracle": { + "meanRecallAt10": 0.979058, + "meanNdcgAt10": 1, + "totalDeliveredTokens": 273777 + } + } + } + }, + "onlinePrequential": { + "tasks": 240, + "positiveUnderStructuralOracle": 209, + "noPositiveUnderStructuralOracle": 31, + "methods": { + "bm25Admitted": { + "tasks": 240, + "scorableTasks": 209, + "meanRecallAt1": 0.116328, + "meanRecallAt5": 0.371323, + "meanRecallAt10": 0.538341, + "meanPrecisionAt5": 0.490909, + "meanPrecisionAt10": 0.423445, + "meanReciprocalRank": 0.695595, + "meanNdcgAt10": 0.557802, + "meanDirectRecallAt10": 0.631341, + "totalEstimatedContextTokens": 1050550 + }, + "balancedV1": { + "tasks": 240, + "scorableTasks": 209, + "meanRecallAt1": 0.080922, + "meanRecallAt5": 0.30058, + "meanRecallAt10": 0.473421, + "meanPrecisionAt5": 0.480383, + "meanPrecisionAt10": 0.42823, + "meanReciprocalRank": 0.600665, + "meanNdcgAt10": 0.507527, + "meanDirectRecallAt10": 0.550948, + "totalEstimatedContextTokens": 976338 + }, + "qarinahV2": { + "tasks": 240, + "scorableTasks": 209, + "meanRecallAt1": 0.116328, + "meanRecallAt5": 0.371323, + "meanRecallAt10": 0.538341, + "meanPrecisionAt5": 0.490909, + "meanPrecisionAt10": 0.423445, + "meanReciprocalRank": 0.695595, + "meanNdcgAt10": 0.557802, + "meanDirectRecallAt10": 0.631341, + "totalEstimatedContextTokens": 1050550 + }, + "qarinahV2NoGraph": { + "tasks": 240, + "scorableTasks": 209, + "meanRecallAt1": 0.116328, + "meanRecallAt5": 0.371323, + "meanRecallAt10": 0.538341, + "meanPrecisionAt5": 0.490909, + "meanPrecisionAt10": 0.423445, + "meanReciprocalRank": 0.695595, + "meanNdcgAt10": 0.557802, + "meanDirectRecallAt10": 0.631341, + "totalEstimatedContextTokens": 1050550 + }, + "oracle": { + "tasks": 240, + "scorableTasks": 209, + "meanRecallAt1": 0.258846, + "meanRecallAt5": 0.652685, + "meanRecallAt10": 0.824015, + "meanPrecisionAt5": 0.813397, + "meanPrecisionAt10": 0.65311, + "meanReciprocalRank": 1, + "meanNdcgAt10": 1, + "meanDirectRecallAt10": 1, + "totalEstimatedContextTokens": 550750 + } + }, + "evidenceSufficiency": { + "positives": 209, + "noPositiveUnderStructuralOracle": 31, + "supported": 187, + "supportedPrecision": 0.871658, + "supportedRecall": 0.779904, + "noPositiveFalseAcceptanceRate": 0.774194, + "noPositiveCorrectAbstentionRate": 0.225806, + "rocAuc": 0.534805, + "prAucAveragePrecision": 0.893086, + "brierScore": 0.268545, + "expectedCalibrationError10Bin": 0.385678, + "riskCoverage": [ + { + "coverage": 0.1, + "threshold": 0.614214, + "risk": 0.041667 + }, + { + "coverage": 0.25, + "threshold": 0.562184, + "risk": 0.133333 + }, + { + "coverage": 0.5, + "threshold": 0.492399, + "risk": 0.125 + }, + { + "coverage": 0.75, + "threshold": 0.41, + "risk": 0.127778 + }, + { + "coverage": 1, + "threshold": 0.175403, + "risk": 0.129167 + } + ] + }, + "directDecision": { + "method": "evidence-sufficiency-v2", + "directThreshold": 0.65, + "partialThreshold": 0.4, + "tasks": 240, + "positives": 209, + "noPositiveUnderStructuralOracle": 31, + "acceptedDirect": 15, + "abstained": 225, + "truePositive": 15, + "falsePositive": 0, + "trueNegative": 31, + "falseNegative": 194, + "acceptedPrecision": 1, + "acceptedRecall": 0.07177, + "acceptedF1": 0.133929, + "falseAcceptanceRate": 0, + "correctAbstentionRate": 1, + "acceptanceCoverage": 0.0625, + "stateCounts": { + "DIRECTLY_SUPPORTED": 15, + "PARTIALLY_SUPPORTED": 172, + "INSUFFICIENT_EVIDENCE": 53 + }, + "confidenceIntervals95": { + "acceptedPrecision": { + "method": "Clopper-Pearson exact two-sided", + "successes": 15, + "trials": 15, + "lower": 0.781981, + "upper": 1 + }, + "falseAcceptanceRate": { + "method": "Clopper-Pearson exact two-sided", + "successes": 0, + "trials": 31, + "lower": 0, + "upper": 0.112189 + } + } + }, + "noTemporalAblation": { + "returnedItems": 2288, + "futureItems": 1083, + "futureItemRate": 0.473339, + "affectedQueries": 240, + "affectedQueryRate": 1 + }, + "budgetCurves": { + "512": { + "bm25Admitted": { + "meanRecallAt10": 0.18211, + "meanNdcgAt10": 0.26622, + "totalDeliveredTokens": 108671 + }, + "balancedV1": { + "meanRecallAt10": 0.145062, + "meanNdcgAt10": 0.228579, + "totalDeliveredTokens": 108628 + }, + "qarinahV2": { + "meanRecallAt10": 0.18211, + "meanNdcgAt10": 0.26622, + "totalDeliveredTokens": 108671 + }, + "qarinahV2NoGraph": { + "meanRecallAt10": 0.18211, + "meanNdcgAt10": 0.26622, + "totalDeliveredTokens": 108671 + }, + "oracle": { + "meanRecallAt10": 0.329223, + "meanNdcgAt10": 0.535114, + "totalDeliveredTokens": 83443 + } + }, + "1000": { + "bm25Admitted": { + "meanRecallAt10": 0.243828, + "meanNdcgAt10": 0.321046, + "totalDeliveredTokens": 218668 + }, + "balancedV1": { + "meanRecallAt10": 0.197006, + "meanNdcgAt10": 0.276441, + "totalDeliveredTokens": 220093 + }, + "qarinahV2": { + "meanRecallAt10": 0.243828, + "meanNdcgAt10": 0.321046, + "totalDeliveredTokens": 218668 + }, + "qarinahV2NoGraph": { + "meanRecallAt10": 0.243828, + "meanNdcgAt10": 0.321046, + "totalDeliveredTokens": 218668 + }, + "oracle": { + "meanRecallAt10": 0.5093, + "meanNdcgAt10": 0.725751, + "totalDeliveredTokens": 166912 + } + }, + "2000": { + "bm25Admitted": { + "meanRecallAt10": 0.358999, + "meanNdcgAt10": 0.416982, + "totalDeliveredTokens": 438819 + }, + "balancedV1": { + "meanRecallAt10": 0.313634, + "meanNdcgAt10": 0.374537, + "totalDeliveredTokens": 439116 + }, + "qarinahV2": { + "meanRecallAt10": 0.358999, + "meanNdcgAt10": 0.416982, + "totalDeliveredTokens": 438819 + }, + "qarinahV2NoGraph": { + "meanRecallAt10": 0.358999, + "meanNdcgAt10": 0.416982, + "totalDeliveredTokens": 438819 + }, + "oracle": { + "meanRecallAt10": 0.671557, + "meanNdcgAt10": 0.873291, + "totalDeliveredTokens": 309235 + } + }, + "4000": { + "bm25Admitted": { + "meanRecallAt10": 0.490415, + "meanNdcgAt10": 0.514965, + "totalDeliveredTokens": 839256 + }, + "balancedV1": { + "meanRecallAt10": 0.420639, + "meanNdcgAt10": 0.465707, + "totalDeliveredTokens": 839351 + }, + "qarinahV2": { + "meanRecallAt10": 0.490415, + "meanNdcgAt10": 0.514965, + "totalDeliveredTokens": 839838 + }, + "qarinahV2NoGraph": { + "meanRecallAt10": 0.490415, + "meanNdcgAt10": 0.514965, + "totalDeliveredTokens": 839838 + }, + "oracle": { + "meanRecallAt10": 0.787273, + "meanNdcgAt10": 0.972814, + "totalDeliveredTokens": 492214 + } + }, + "8000": { + "bm25Admitted": { + "meanRecallAt10": 0.531145, + "meanNdcgAt10": 0.550819, + "totalDeliveredTokens": 1528888 + }, + "balancedV1": { + "meanRecallAt10": 0.467711, + "meanNdcgAt10": 0.502644, + "totalDeliveredTokens": 1538598 + }, + "qarinahV2": { + "meanRecallAt10": 0.531145, + "meanNdcgAt10": 0.550819, + "totalDeliveredTokens": 1531828 + }, + "qarinahV2NoGraph": { + "meanRecallAt10": 0.531145, + "meanNdcgAt10": 0.550819, + "totalDeliveredTokens": 1531828 + }, + "oracle": { + "meanRecallAt10": 0.824015, + "meanNdcgAt10": 1, + "totalDeliveredTokens": 737279 + } + } + } + } + }, + "inference": [ + { + "method": "repository-clustered-bootstrap", + "samples": 10000, + "clusters": 12, + "pairedTasks": 209, + "left": "qarinahV2", + "right": "bm25Admitted", + "metric": "recallAt10", + "observedMeanDifference": 0, + "confidenceInterval95": [ + 0, + 0 + ] + }, + { + "method": "repository-clustered-bootstrap", + "samples": 10000, + "clusters": 12, + "pairedTasks": 209, + "left": "qarinahV2", + "right": "balancedV1", + "metric": "recallAt10", + "observedMeanDifference": 0.064919, + "confidenceInterval95": [ + -0.015019, + 0.102501 + ] + }, + { + "method": "repository-clustered-bootstrap", + "samples": 10000, + "clusters": 12, + "pairedTasks": 209, + "left": "qarinahV2", + "right": "balancedV1", + "metric": "reciprocalRank", + "observedMeanDifference": 0.09493, + "confidenceInterval95": [ + 0.057183, + 0.111456 + ] + } + ], + "taskResults": { + "static": [ + { + "repository": "astropy/astropy", + "instanceId": "astropy__astropy-7746", + "repositorySequence": 2, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 187, + "balancedV1": 187, + "qarinahV2": 187, + "qarinahV2NoGraph": 187, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 187, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 187, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 187, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 187, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 187, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 187, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 187, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 187, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 187, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 187, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.261375, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 6 + }, + { + "repository": "astropy/astropy", + "instanceId": "astropy__astropy-12907", + "repositorySequence": 3, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 187, + "balancedV1": 187, + "qarinahV2": 187, + "qarinahV2NoGraph": 187, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 187, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 187, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 187, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 187, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 187, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 187, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 187, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 187, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 187, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 187, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.291071, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 6 + }, + { + "repository": "astropy/astropy", + "instanceId": "astropy__astropy-14182", + "repositorySequence": 4, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 187, + "balancedV1": 187, + "qarinahV2": 187, + "qarinahV2NoGraph": 187, + "oracle": 187 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.256357, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 6 + }, + { + "repository": "astropy/astropy", + "instanceId": "astropy__astropy-14365", + "repositorySequence": 5, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 187, + "balancedV1": 187, + "qarinahV2": 187, + "qarinahV2NoGraph": 187, + "oracle": 187 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.309239, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 6 + }, + { + "repository": "astropy/astropy", + "instanceId": "astropy__astropy-14995", + "repositorySequence": 6, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 187, + "balancedV1": 187, + "qarinahV2": 187, + "qarinahV2NoGraph": 187, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 187, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 187, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 187, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 187, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 187, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 187, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 187, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 187, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 187, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 187, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.241709, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 6 + }, + { + "repository": "django/django", + "instanceId": "django__django-12113", + "repositorySequence": 24, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.456997, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.636364, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 0.5, + "ndcgAt10": 0.631144, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.456997, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.456997, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 4621, + "balancedV1": 3714, + "qarinahV2": 4621, + "qarinahV2NoGraph": 4621, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 881, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 975, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2": { + "deliveredTokens": 881, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 881, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1870, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.31488, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 1980, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.343697, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2": { + "deliveredTokens": 1870, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.31488, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1870, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.31488, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3948, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.454545, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.526428, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "balancedV1": { + "deliveredTokens": 3947, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.636364, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 0.5, + "ndcgAt10": 0.631144, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 3948, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.454545, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.526428, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3948, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.454545, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.526428, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.456997, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.636364, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 0.5, + "ndcgAt10": 0.631144, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.456997, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.456997, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.371261, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 9, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-12125", + "repositorySequence": 25, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 10, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.756234, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + }, + "balancedV1": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.727273, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.644136, + "directRecallAt10": 1, + "supportingRecallAt10": 0.727273 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.756234, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.756234, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 4772, + "balancedV1": 3512, + "qarinahV2": 4772, + "qarinahV2NoGraph": 4772, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.458466, + "directRecallAt10": 1, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.458466, + "directRecallAt10": 1, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.458466, + "directRecallAt10": 1, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.458466, + "directRecallAt10": 1, + "supportingRecallAt10": 0.090909 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 954, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 997, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.374178, + "directRecallAt10": 0, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2": { + "deliveredTokens": 954, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 954, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.631297, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1964, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.697114, + "directRecallAt10": 1, + "supportingRecallAt10": 0.363636 + } + }, + "balancedV1": { + "deliveredTokens": 1962, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 1964, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.697114, + "directRecallAt10": 1, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1964, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.697114, + "directRecallAt10": 1, + "supportingRecallAt10": 0.363636 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.756234, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3874, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.756234, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "balancedV1": { + "deliveredTokens": 3938, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.727273, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.644136, + "directRecallAt10": 1, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 3874, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.756234, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3874, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.756234, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.756234, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.727273, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.644136, + "directRecallAt10": 1, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.756234, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.756234, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.507195, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-12184", + "repositorySequence": 26, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 4445, + "balancedV1": 3634, + "qarinahV2": 4445, + "qarinahV2NoGraph": 4445, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 452, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 419, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 452, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 452, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 943, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 885, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 943, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 943, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1882, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1890, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1882, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1882, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3969, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3940, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3969, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3969, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.543478, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 10, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-12284", + "repositorySequence": 27, + "positiveUnderStructuralOracle": true, + "directRecords": 4, + "supportingRecords": 7, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.618073, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.636364 + }, + "balancedV1": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.636364, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.852483, + "directRecallAt10": 1, + "supportingRecallAt10": 0.636364 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.618073, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.636364 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.618073, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.636364 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 3780, + "balancedV1": 3445, + "qarinahV2": 3780, + "qarinahV2NoGraph": 3780, + "oracle": 3195 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.310341, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.53078, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.310341, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.310341, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.506145, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 941, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.362065, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.784493, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2": { + "deliveredTokens": 941, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.362065, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 941, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.362065, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 955, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.834991, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1951, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.446636, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.363636 + } + }, + "balancedV1": { + "deliveredTokens": 1916, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.636364, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.853976, + "directRecallAt10": 1, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 1951, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.446636, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1951, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.446636, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.363636 + } + }, + "oracle": { + "deliveredTokens": 1933, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.727273, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.938956, + "directRecallAt10": 1, + "supportingRecallAt10": 0.727273 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3918, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.618073, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.636364 + } + }, + "balancedV1": { + "deliveredTokens": 3951, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.636364, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.852483, + "directRecallAt10": 1, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 3918, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.618073, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3918, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.618073, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.636364 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.618073, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.636364 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.636364, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.852483, + "directRecallAt10": 1, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.618073, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.618073, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.636364 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.589706, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-12286", + "repositorySequence": 28, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3545, + "balancedV1": 3445, + "qarinahV2": 3545, + "qarinahV2NoGraph": 3545, + "oracle": 523 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 394, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 233, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1872, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 1890, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1872, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1872, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3989, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 3929, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3989, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3989, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.453571, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-12308", + "repositorySequence": 29, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.395647, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.395647, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.395647, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3535, + "balancedV1": 3600, + "qarinahV2": 3535, + "qarinahV2NoGraph": 3535, + "oracle": 626 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 251, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 939, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 939, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 939, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1882, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1967, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1882, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1882, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.395647, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3982, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.395647, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.395647, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.395647, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.395647, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.395647, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.451429, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 10, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-12453", + "repositorySequence": 30, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.272727, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.315163, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.636364, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 0.5, + "ndcgAt10": 0.621499, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.272727, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.315163, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.272727, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.315163, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 4730, + "balancedV1": 3595, + "qarinahV2": 4730, + "qarinahV2NoGraph": 4730, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 421, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 421, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 421, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 904, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 885, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.453743, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2": { + "deliveredTokens": 904, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 904, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1891, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.248908, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 1962, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.248908, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 1891, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.248908, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1891, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.248908, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3937, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.272727, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.315163, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 3901, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.636364, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 0.5, + "ndcgAt10": 0.621499, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 3937, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.272727, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.315163, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3937, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.272727, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.315163, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.272727, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.315163, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.636364, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 0.5, + "ndcgAt10": 0.621499, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.272727, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.315163, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.272727, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.315163, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.374653, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-12470", + "repositorySequence": 31, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 10, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.636364, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.698964, + "directRecallAt10": 1, + "supportingRecallAt10": 0.636364 + }, + "balancedV1": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.545455, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.498799, + "directRecallAt10": 0, + "supportingRecallAt10": 0.545455 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.636364, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.698964, + "directRecallAt10": 1, + "supportingRecallAt10": 0.636364 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.636364, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.698964, + "directRecallAt10": 1, + "supportingRecallAt10": 0.636364 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 3343, + "balancedV1": 3482, + "qarinahV2": 3343, + "qarinahV2NoGraph": 3343, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 992, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.518493, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 991, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.450589, + "directRecallAt10": 0, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2": { + "deliveredTokens": 992, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.518493, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 992, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.518493, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 982, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.631297, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1914, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.501202, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 1972, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.545455, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.50153, + "directRecallAt10": 0, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2": { + "deliveredTokens": 1914, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.501202, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1914, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.501202, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.756234, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.636364, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.698964, + "directRecallAt10": 1, + "supportingRecallAt10": 0.636364 + } + }, + "balancedV1": { + "deliveredTokens": 3919, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.545455, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.498799, + "directRecallAt10": 0, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2": { + "deliveredTokens": 3934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.636364, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.698964, + "directRecallAt10": 1, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.636364, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.698964, + "directRecallAt10": 1, + "supportingRecallAt10": 0.636364 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.636364, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.698964, + "directRecallAt10": 1, + "supportingRecallAt10": 0.636364 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.545455, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.498799, + "directRecallAt10": 0, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.636364, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.698964, + "directRecallAt10": 1, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.636364, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.698964, + "directRecallAt10": 1, + "supportingRecallAt10": 0.636364 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.5875, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-12497", + "repositorySequence": 32, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.454545, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.637152, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + }, + "balancedV1": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.747935, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.454545, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.637152, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.454545, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.637152, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 4246, + "balancedV1": 3440, + "qarinahV2": 4246, + "qarinahV2NoGraph": 4246, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 500, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 500, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 500, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 959, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2": { + "deliveredTokens": 959, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 959, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1946, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "balancedV1": { + "deliveredTokens": 1871, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.453743, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2": { + "deliveredTokens": 1946, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1946, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3940, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.454545, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.637152, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "balancedV1": { + "deliveredTokens": 3888, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.747935, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 3940, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.454545, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.637152, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3940, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.454545, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.637152, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.454545, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.637152, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.747935, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.454545, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.637152, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.454545, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.637152, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.487879, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 9, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-12589", + "repositorySequence": 33, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.769526, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + }, + "balancedV1": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.782739, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.769526, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.769526, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 4511, + "balancedV1": 3512, + "qarinahV2": 4511, + "qarinahV2NoGraph": 4511, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 404, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 404, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 404, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 904, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 903, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2": { + "deliveredTokens": 904, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 904, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1977, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.554143, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "balancedV1": { + "deliveredTokens": 1900, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.424926, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2": { + "deliveredTokens": 1977, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.554143, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1977, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.554143, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3923, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.77216, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "balancedV1": { + "deliveredTokens": 3938, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.782739, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 3923, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.77216, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3923, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.77216, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.769526, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.782739, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.769526, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.769526, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.448293, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-12700", + "repositorySequence": 34, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.289065, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4613, + "balancedV1": 3600, + "qarinahV2": 4613, + "qarinahV2NoGraph": 4613, + "oracle": 295 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 449, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 295, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 909, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 909, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 909, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 295, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1976, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1978, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1976, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1976, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 295, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3921, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3906, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.289065, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3921, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3921, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 295, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.289065, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 295, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.419298, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-12708", + "repositorySequence": 35, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.545455, + "precisionAt5": 0.4, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.592683, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.636364, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 0.5, + "ndcgAt10": 0.628511, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.545455, + "precisionAt5": 0.4, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.592683, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.545455, + "precisionAt5": 0.4, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.592683, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 4489, + "balancedV1": 3714, + "qarinahV2": 4489, + "qarinahV2NoGraph": 4489, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 502, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.330138, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 400, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 502, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.330138, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 502, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.330138, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 877, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.31488, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 997, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.554143, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2": { + "deliveredTokens": 877, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.31488, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 877, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.31488, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1973, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 1871, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.343697, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2": { + "deliveredTokens": 1973, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1973, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3989, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.545455, + "precisionAt5": 0.4, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.592683, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "balancedV1": { + "deliveredTokens": 3947, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.636364, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 0.5, + "ndcgAt10": 0.628511, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 3989, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.545455, + "precisionAt5": 0.4, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.592683, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3989, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.545455, + "precisionAt5": 0.4, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.592683, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.545455, + "precisionAt5": 0.4, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.592683, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.636364, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 0.5, + "ndcgAt10": 0.628511, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.545455, + "precisionAt5": 0.4, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.592683, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.545455, + "precisionAt5": 0.4, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.592683, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.515385, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 9, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-12747", + "repositorySequence": 36, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 10, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.732053, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + }, + "balancedV1": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.534325, + "directRecallAt10": 0, + "supportingRecallAt10": 0.636364 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.732053, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.732053, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 4570, + "balancedV1": 3634, + "qarinahV2": 4570, + "qarinahV2NoGraph": 4570, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 426, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.458466, + "directRecallAt10": 1, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 426, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.458466, + "directRecallAt10": 1, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 426, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.458466, + "directRecallAt10": 1, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.631297, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 988, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.524283, + "directRecallAt10": 1, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.450589, + "directRecallAt10": 0, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2": { + "deliveredTokens": 988, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.524283, + "directRecallAt10": 1, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 988, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.524283, + "directRecallAt10": 1, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 984, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.697114, + "directRecallAt10": 1, + "supportingRecallAt10": 0.363636 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1908, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.583403, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 1871, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.325653, + "directRecallAt10": 0, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2": { + "deliveredTokens": 1908, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.583403, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1908, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.583403, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 1933, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.727273, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.909821, + "directRecallAt10": 1, + "supportingRecallAt10": 0.727273 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3923, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.636364, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.776228, + "directRecallAt10": 1, + "supportingRecallAt10": 0.636364 + } + }, + "balancedV1": { + "deliveredTokens": 3980, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.534325, + "directRecallAt10": 0, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 3923, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.636364, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.776228, + "directRecallAt10": 1, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3923, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.636364, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.776228, + "directRecallAt10": 1, + "supportingRecallAt10": 0.636364 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.732053, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.534325, + "directRecallAt10": 0, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.732053, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.732053, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.3865, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 8, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-12856", + "repositorySequence": 37, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.55197, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + }, + "balancedV1": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.696938, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.55197, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.55197, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 3070, + "balancedV1": 3287, + "qarinahV2": 3070, + "qarinahV2NoGraph": 3070, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 444, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 400, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 444, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 444, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 996, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.248908, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 997, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.554143, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2": { + "deliveredTokens": 996, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.248908, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 996, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.248908, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1881, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.363636, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.422095, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "balancedV1": { + "deliveredTokens": 1962, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.705905, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2": { + "deliveredTokens": 1881, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.363636, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.422095, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1881, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.363636, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.422095, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3789, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.55197, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "balancedV1": { + "deliveredTokens": 3961, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.696938, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2": { + "deliveredTokens": 3789, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.55197, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3789, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.55197, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.55197, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.696938, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.55197, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.55197, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.52375, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-12908", + "repositorySequence": 38, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.752969, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + }, + "balancedV1": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.831848, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.752969, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.752969, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 3308, + "balancedV1": 3798, + "qarinahV2": 3308, + "qarinahV2NoGraph": 3308, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 404, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 881, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 895, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2": { + "deliveredTokens": 881, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 881, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1875, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.454545, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.617284, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "balancedV1": { + "deliveredTokens": 1916, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.775337, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 1875, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.454545, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.617284, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1875, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.454545, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.617284, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3986, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.752969, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "balancedV1": { + "deliveredTokens": 3929, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.831848, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 3986, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.752969, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3986, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.752969, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.752969, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.831848, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.752969, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.752969, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.4425, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 8, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-12915", + "repositorySequence": 39, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.218407, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.177239, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.218407, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.218407, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4702, + "balancedV1": 3714, + "qarinahV2": 4702, + "qarinahV2NoGraph": 4702, + "oracle": 626 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 437, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 449, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 437, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 437, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 251, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 954, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 915, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 954, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 954, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1999, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1912, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1999, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1999, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3957, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.218407, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 3990, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.177239, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 3957, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.218407, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3957, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.218407, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.218407, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.177239, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.218407, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.218407, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.566384, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-12983", + "repositorySequence": 40, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.25, + "ndcgAt10": 0.4858, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.845185, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.25, + "ndcgAt10": 0.4858, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.25, + "ndcgAt10": 0.4858, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4665, + "balancedV1": 3365, + "qarinahV2": 4665, + "qarinahV2NoGraph": 4665, + "oracle": 1678 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 386, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 444, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 943, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 885, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 943, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 943, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 444, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1992, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.202107, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 1882, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.703918, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 1992, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.202107, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1992, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.202107, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 1678, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3898, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.25, + "ndcgAt10": 0.350148, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 3950, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.845185, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3898, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.25, + "ndcgAt10": 0.350148, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3898, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.25, + "ndcgAt10": 0.350148, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 1678, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.25, + "ndcgAt10": 0.4858, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.845185, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.25, + "ndcgAt10": 0.4858, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.25, + "ndcgAt10": 0.4858, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1678, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.528571, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13028", + "repositorySequence": 41, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.820523, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.706544, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.820523, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.820523, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 4403, + "balancedV1": 3701, + "qarinahV2": 4403, + "qarinahV2NoGraph": 4403, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 419, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 978, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 991, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2": { + "deliveredTokens": 978, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 978, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1978, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "balancedV1": { + "deliveredTokens": 1900, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.343697, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2": { + "deliveredTokens": 1978, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1978, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3983, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.760079, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "balancedV1": { + "deliveredTokens": 3934, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.706544, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 3983, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.760079, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3983, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.760079, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.820523, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.706544, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.820523, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.820523, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.429682, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 9, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13033", + "repositorySequence": 42, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 10, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.791328, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + }, + "balancedV1": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.655517, + "directRecallAt10": 1, + "supportingRecallAt10": 0.727273 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.791328, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.791328, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 4028, + "balancedV1": 3953, + "qarinahV2": 4028, + "qarinahV2NoGraph": 4028, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 484, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 976, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 903, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.39147, + "directRecallAt10": 0, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2": { + "deliveredTokens": 976, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 976, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 982, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.631297, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1871, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.697114, + "directRecallAt10": 1, + "supportingRecallAt10": 0.363636 + } + }, + "balancedV1": { + "deliveredTokens": 1980, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.315059, + "directRecallAt10": 0, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2": { + "deliveredTokens": 1871, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.697114, + "directRecallAt10": 1, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1871, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.697114, + "directRecallAt10": 1, + "supportingRecallAt10": 0.363636 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.756234, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3963, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.791328, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + } + }, + "balancedV1": { + "deliveredTokens": 3953, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.655517, + "directRecallAt10": 1, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 3963, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.791328, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3963, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.791328, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.791328, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.655517, + "directRecallAt10": 1, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.791328, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.791328, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.429664, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13158", + "repositorySequence": 43, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.545455, + "precisionAt5": 0.4, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.631624, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + }, + "balancedV1": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.636364, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.78855, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.545455, + "precisionAt5": 0.4, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.631624, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.545455, + "precisionAt5": 0.4, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.631624, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 3351, + "balancedV1": 3820, + "qarinahV2": 3351, + "qarinahV2NoGraph": 3351, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 498, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 434, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 498, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 498, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 992, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 933, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2": { + "deliveredTokens": 992, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 992, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1917, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 1972, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.545455, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2": { + "deliveredTokens": 1917, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1917, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3933, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.545455, + "precisionAt5": 0.4, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.631624, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "balancedV1": { + "deliveredTokens": 3951, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.636364, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.78855, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 3933, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.545455, + "precisionAt5": 0.4, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.631624, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3933, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.545455, + "precisionAt5": 0.4, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.631624, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.545455, + "precisionAt5": 0.4, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.631624, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.636364, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.78855, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.545455, + "precisionAt5": 0.4, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.631624, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.545455, + "precisionAt5": 0.4, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.631624, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.405769, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13220", + "repositorySequence": 44, + "positiveUnderStructuralOracle": true, + "directRecords": 2, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.75, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.53718, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.75, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.53718, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.75, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.53718, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + }, + "oracle": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4506, + "balancedV1": 3201, + "qarinahV2": 4506, + "qarinahV2NoGraph": 4506, + "oracle": 1686 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 510, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 510, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 510, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 233, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.171719, + "directRecallAt10": 0, + "supportingRecallAt10": 0.25 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 943, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.325028, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 943, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.325028, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 943, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.325028, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 881, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.6235, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.325028, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 1995, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.325028, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.325028, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 1686, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3924, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.75, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.539235, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + }, + "balancedV1": { + "deliveredTokens": 3991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3924, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.75, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.539235, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3924, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.75, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.539235, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + }, + "oracle": { + "deliveredTokens": 1686, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.75, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.53718, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.75, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.53718, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.75, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.53718, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + }, + "oracle": { + "deliveredTokens": 1686, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.468868, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13230", + "repositorySequence": 45, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3922, + "balancedV1": 3296, + "qarinahV2": 3922, + "qarinahV2NoGraph": 3922, + "oracle": 626 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 437, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 437, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 437, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 251, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 976, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 948, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 976, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 976, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1880, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 1995, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1880, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1880, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3922, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 3892, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3922, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3922, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.355, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "NO_CODE_ENTITY_MATCH", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13265", + "repositorySequence": 46, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 10, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.770002, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + }, + "balancedV1": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.614837, + "directRecallAt10": 1, + "supportingRecallAt10": 0.636364 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.770002, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.770002, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 3265, + "balancedV1": 3428, + "qarinahV2": 3265, + "qarinahV2NoGraph": 3265, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 500, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 500, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 500, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 959, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.631297, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 884, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.39147, + "directRecallAt10": 0, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2": { + "deliveredTokens": 959, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.631297, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 959, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.631297, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 992, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.697114, + "directRecallAt10": 1, + "supportingRecallAt10": 0.363636 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1986, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.620703, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 1871, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.315059, + "directRecallAt10": 0, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2": { + "deliveredTokens": 1986, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.620703, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1986, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.620703, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.756234, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3913, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.770002, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + } + }, + "balancedV1": { + "deliveredTokens": 3887, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.614837, + "directRecallAt10": 1, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 3913, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.770002, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3913, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.770002, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.770002, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.614837, + "directRecallAt10": 1, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.770002, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.770002, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.494878, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13315", + "repositorySequence": 47, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.356207, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.356207, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.356207, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4810, + "balancedV1": 3073, + "qarinahV2": 4810, + "qarinahV2NoGraph": 4810, + "oracle": 515 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 0, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1978, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1871, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1978, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1978, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.356207, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3950, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.356207, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.356207, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.356207, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.356207, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.356207, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.45, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "SINGLE_OR_NO_SUPPORT", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 9, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13321", + "repositorySequence": 48, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.177239, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4540, + "balancedV1": 3680, + "qarinahV2": 4540, + "qarinahV2NoGraph": 4540, + "oracle": 626 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 510, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 394, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 510, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 510, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 251, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 976, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 903, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 976, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 976, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1904, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 1890, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1904, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1904, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3892, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 3975, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.177239, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 3892, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3892, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.177239, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.365202, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13401", + "repositorySequence": 49, + "positiveUnderStructuralOracle": true, + "directRecords": 4, + "supportingRecords": 7, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.363636, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.32562, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.363636 + }, + "balancedV1": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.909532, + "directRecallAt10": 1, + "supportingRecallAt10": 0.727273 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.363636, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.32562, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.363636 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.363636, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.32562, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.363636 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 3476, + "balancedV1": 3953, + "qarinahV2": 3476, + "qarinahV2NoGraph": 3476, + "oracle": 3195 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.195804, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.557869, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2": { + "deliveredTokens": 433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.195804, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.195804, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.506145, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 943, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.103447, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.834991, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2": { + "deliveredTokens": 943, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.103447, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 943, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.103447, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 955, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.834991, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1985, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.288771, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 1871, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.661316, + "directRecallAt10": 0.75, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2": { + "deliveredTokens": 1985, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.288771, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1985, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.288771, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 1933, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.727273, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.938956, + "directRecallAt10": 1, + "supportingRecallAt10": 0.727273 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3975, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.363636, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.32562, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.363636 + } + }, + "balancedV1": { + "deliveredTokens": 3953, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.909532, + "directRecallAt10": 1, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 3975, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.363636, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.32562, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3975, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.363636, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.32562, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.363636 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.363636, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.32562, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.363636 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.909532, + "directRecallAt10": 1, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.363636, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.32562, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.363636, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.32562, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.363636 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.420714, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13447", + "repositorySequence": 50, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4022, + "balancedV1": 3201, + "qarinahV2": 4022, + "qarinahV2NoGraph": 4022, + "oracle": 626 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 510, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 510, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 510, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 251, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 954, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 954, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 954, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1883, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 1871, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1883, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1883, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3985, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 3966, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3985, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3985, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.4375, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "SINGLE_OR_NO_SUPPORT", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13448", + "repositorySequence": 51, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.337208, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.727273, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.669862, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.337208, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.337208, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 4793, + "balancedV1": 3953, + "qarinahV2": 4793, + "qarinahV2NoGraph": 4793, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 505, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2": { + "deliveredTokens": 505, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 505, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 938, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 915, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.424926, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2": { + "deliveredTokens": 938, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 938, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1950, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.094788, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 1962, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.233651, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 1950, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.094788, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1950, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.094788, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3977, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.343018, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "balancedV1": { + "deliveredTokens": 3953, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.727273, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.669862, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 3977, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.343018, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3977, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.343018, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.337208, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.727273, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.669862, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.337208, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.337208, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.455681, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13551", + "repositorySequence": 52, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2610, + "balancedV1": 3219, + "qarinahV2": 2610, + "qarinahV2NoGraph": 2610, + "oracle": 626 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 477, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 477, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 477, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 251, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 956, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 948, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 956, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 956, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1925, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1985, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1925, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1925, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3924, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 3946, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3924, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3924, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.22, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "NO_CODE_ENTITY_MATCH", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13590", + "repositorySequence": 53, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 0.5, + "ndcgAt10": 0.646856, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + }, + "balancedV1": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.779171, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 0.5, + "ndcgAt10": 0.646856, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 0.5, + "ndcgAt10": 0.646856, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 4386, + "balancedV1": 3383, + "qarinahV2": 4386, + "qarinahV2NoGraph": 4386, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 495, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 419, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 495, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 495, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 985, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.248908, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2": { + "deliveredTokens": 985, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.248908, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 985, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.248908, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1972, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.363636, + "recallAt10": 0.454545, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.507238, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "balancedV1": { + "deliveredTokens": 1985, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.545455, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2": { + "deliveredTokens": 1972, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.363636, + "recallAt10": 0.454545, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.507238, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1972, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.363636, + "recallAt10": 0.454545, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.507238, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3889, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.710477, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "balancedV1": { + "deliveredTokens": 3988, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.779171, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 3889, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.710477, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3889, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.710477, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 0.5, + "ndcgAt10": 0.646856, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.779171, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 0.5, + "ndcgAt10": 0.646856, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 0.5, + "ndcgAt10": 0.646856, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.605469, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13658", + "repositorySequence": 54, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.411834, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.411834, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.411834, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4570, + "balancedV1": 3611, + "qarinahV2": 4570, + "qarinahV2NoGraph": 4570, + "oracle": 523 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 233, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 954, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 915, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 954, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 954, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1955, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1962, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1955, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1955, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3922, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.411834, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3901, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3922, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.411834, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3922, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.411834, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.411834, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.411834, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.411834, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.577778, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13660", + "repositorySequence": 55, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.44158, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.44158, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.44158, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4599, + "balancedV1": 3953, + "qarinahV2": 4599, + "qarinahV2NoGraph": 4599, + "oracle": 523 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 233, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 904, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 915, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 904, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 904, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1979, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1962, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1979, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1979, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.44158, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3953, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.44158, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.44158, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.44158, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.44158, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.44158, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.603153, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13710", + "repositorySequence": 56, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.152733, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.069976, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.152733, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.152733, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4424, + "balancedV1": 3237, + "qarinahV2": 4424, + "qarinahV2NoGraph": 4424, + "oracle": 1240 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 510, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 510, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 510, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 251, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.242076, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.152733, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.152733, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.152733, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 865, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1990, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.152733, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 1871, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1990, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.152733, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1990, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.152733, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 1240, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.152733, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 3985, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.069976, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 3991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.152733, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.152733, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 1240, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.152733, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.069976, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.152733, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.152733, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 1240, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.425893, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "SINGLE_OR_NO_SUPPORT", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13757", + "repositorySequence": 57, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 10, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.549339, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + }, + "balancedV1": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.686245, + "directRecallAt10": 1, + "supportingRecallAt10": 0.636364 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.549339, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.549339, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 3248, + "balancedV1": 3556, + "qarinahV2": 3248, + "qarinahV2NoGraph": 3248, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 447, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 394, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 447, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 447, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 484, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 986, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 903, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.544292, + "directRecallAt10": 1, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2": { + "deliveredTokens": 986, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 986, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 943, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.631297, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1985, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.363636, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.497633, + "directRecallAt10": 1, + "supportingRecallAt10": 0.363636 + } + }, + "balancedV1": { + "deliveredTokens": 1875, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.478475, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2": { + "deliveredTokens": 1985, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.363636, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.497633, + "directRecallAt10": 1, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1985, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.363636, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.497633, + "directRecallAt10": 1, + "supportingRecallAt10": 0.363636 + } + }, + "oracle": { + "deliveredTokens": 1993, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.727273, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.909821, + "directRecallAt10": 1, + "supportingRecallAt10": 0.727273 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3922, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.549339, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "balancedV1": { + "deliveredTokens": 3977, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.686245, + "directRecallAt10": 1, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 3922, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.549339, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3922, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.549339, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.549339, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.686245, + "directRecallAt10": 1, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.549339, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.549339, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.4475, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13768", + "repositorySequence": 58, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 4006, + "balancedV1": 3615, + "qarinahV2": 4006, + "qarinahV2NoGraph": 4006, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 441, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 466, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 441, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 441, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 924, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 948, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 924, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 924, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1989, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1887, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1989, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1989, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3925, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3910, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3925, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3925, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.219828, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "NO_CODE_ENTITY_MATCH", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13925", + "repositorySequence": 59, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.454545, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.544221, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + }, + "balancedV1": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.750336, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.454545, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.544221, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.454545, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.544221, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 4415, + "balancedV1": 3714, + "qarinahV2": 4415, + "qarinahV2NoGraph": 4415, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 910, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2": { + "deliveredTokens": 910, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 910, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1890, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.330138, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 1890, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.453743, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2": { + "deliveredTokens": 1890, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.330138, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1890, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.330138, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3952, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.545455, + "precisionAt5": 0.4, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.607842, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "balancedV1": { + "deliveredTokens": 3947, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.750336, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 3952, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.545455, + "precisionAt5": 0.4, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.607842, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3952, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.545455, + "precisionAt5": 0.4, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.607842, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.454545, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.544221, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.750336, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.454545, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.544221, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.454545, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.544221, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.375246, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13933", + "repositorySequence": 60, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3364, + "balancedV1": 3445, + "qarinahV2": 3364, + "qarinahV2NoGraph": 3364, + "oracle": 515 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 426, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 394, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 426, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 426, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 0, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1927, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1883, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1927, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1927, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3879, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 3951, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3879, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3879, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.509836, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13964", + "repositorySequence": 61, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.577149, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + }, + "balancedV1": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.826813, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.577149, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.577149, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 3654, + "balancedV1": 3512, + "qarinahV2": 3654, + "qarinahV2NoGraph": 3654, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 500, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 400, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 500, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 500, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 959, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2": { + "deliveredTokens": 959, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 959, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.272727, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.437352, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 1871, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.272727, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.437352, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.272727, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.437352, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3972, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.577149, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "balancedV1": { + "deliveredTokens": 3933, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.826813, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 3972, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.577149, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3972, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.577149, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.577149, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.826813, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.577149, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.577149, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.387143, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-14016", + "repositorySequence": 62, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.58875, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + }, + "balancedV1": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.820523, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.58875, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.58875, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 3582, + "balancedV1": 3798, + "qarinahV2": 3582, + "qarinahV2NoGraph": 3582, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 938, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2": { + "deliveredTokens": 938, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 938, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1959, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.444097, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 1978, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.538886, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2": { + "deliveredTokens": 1959, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.444097, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1959, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.444097, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3928, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.58875, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "balancedV1": { + "deliveredTokens": 3929, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.820523, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 3928, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.58875, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3928, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.58875, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.58875, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.820523, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.58875, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.58875, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "DIRECTLY_SUPPORTED", + "decision": "ACCEPT_DIRECT", + "score": 0.696939, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "CONSERVATIVE_DIRECT_THRESHOLD_MET" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-14017", + "repositorySequence": 63, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 10, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.486586, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.579106, + "directRecallAt10": 1, + "supportingRecallAt10": 0.727273 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.486586, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.486586, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 4771, + "balancedV1": 3764, + "qarinahV2": 4771, + "qarinahV2NoGraph": 4771, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 424, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 952, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.229233, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 948, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.39147, + "directRecallAt10": 0, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2": { + "deliveredTokens": 952, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.229233, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 952, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.229233, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 982, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.631297, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1999, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.09642, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 1953, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.297767, + "directRecallAt10": 0, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2": { + "deliveredTokens": 1999, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.09642, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1999, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.09642, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.756234, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3881, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.490621, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "balancedV1": { + "deliveredTokens": 3979, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.579106, + "directRecallAt10": 1, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 3881, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.490621, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3881, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.490621, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.486586, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.579106, + "directRecallAt10": 1, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.486586, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.486586, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.4775, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-14155", + "repositorySequence": 64, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4267, + "balancedV1": 3600, + "qarinahV2": 4267, + "qarinahV2NoGraph": 4267, + "oracle": 515 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 414, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 414, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 414, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 0, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 890, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 948, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 890, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 890, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1995, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 3890, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.41, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-14238", + "repositorySequence": 65, + "positiveUnderStructuralOracle": true, + "directRecords": 4, + "supportingRecords": 7, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.371932, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.545455 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.727273, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.688223, + "directRecallAt10": 1, + "supportingRecallAt10": 0.727273 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.371932, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.545455 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.371932, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.545455 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 4603, + "balancedV1": 3798, + "qarinahV2": 4603, + "qarinahV2NoGraph": 4603, + "oracle": 3195 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 495, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.195804, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.310341, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2": { + "deliveredTokens": 495, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.195804, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 495, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.195804, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.506145, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 881, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.103447, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 915, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.599169, + "directRecallAt10": 0.75, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2": { + "deliveredTokens": 881, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.103447, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 881, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.103447, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 955, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.834991, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1882, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.065268, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 1962, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.32946, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 1882, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.065268, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1882, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.065268, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 1933, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.727273, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.938956, + "directRecallAt10": 1, + "supportingRecallAt10": 0.727273 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3925, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.363636, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.184321, + "directRecallAt10": 0, + "supportingRecallAt10": 0.363636 + } + }, + "balancedV1": { + "deliveredTokens": 3929, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.727273, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.688223, + "directRecallAt10": 1, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 3925, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.363636, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.184321, + "directRecallAt10": 0, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3925, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.363636, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.184321, + "directRecallAt10": 0, + "supportingRecallAt10": 0.363636 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.371932, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.545455 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.727273, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.688223, + "directRecallAt10": 1, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.371932, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.371932, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.545455 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.443589, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-14382", + "repositorySequence": 66, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.468076, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.079612, + "directRecallAt10": 0, + "supportingRecallAt10": 0.5 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.468076, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.468076, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4116, + "balancedV1": 3219, + "qarinahV2": 4116, + "qarinahV2NoGraph": 4116, + "oracle": 523 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 484, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.275412, + "directRecallAt10": 0, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 484, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.275412, + "directRecallAt10": 0, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 484, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.275412, + "directRecallAt10": 0, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 290, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.826235, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 917, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.275412, + "directRecallAt10": 0, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 917, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.275412, + "directRecallAt10": 0, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 917, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.275412, + "directRecallAt10": 0, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1900, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.173765, + "directRecallAt10": 0, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 1871, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1900, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.173765, + "directRecallAt10": 0, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1900, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.173765, + "directRecallAt10": 0, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3978, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.468076, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3891, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.079612, + "directRecallAt10": 0, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 3978, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.468076, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3978, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.468076, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.468076, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.079612, + "directRecallAt10": 0, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.468076, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.468076, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.614214, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-14411", + "repositorySequence": 67, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4657, + "balancedV1": 3365, + "qarinahV2": 4657, + "qarinahV2NoGraph": 4657, + "oracle": 626 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 251, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 969, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 948, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 969, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 969, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1908, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1953, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1908, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1908, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3871, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 3966, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3871, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3871, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.55, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-14534", + "repositorySequence": 68, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3767, + "balancedV1": 3600, + "qarinahV2": 3767, + "qarinahV2NoGraph": 3767, + "oracle": 515 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 0, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 943, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 915, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 943, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 943, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1964, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1920, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1964, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1964, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3954, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 3975, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3954, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3954, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.346667, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 8, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-14580", + "repositorySequence": 69, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 10, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.68346, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + }, + "balancedV1": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.727273, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.640479, + "directRecallAt10": 1, + "supportingRecallAt10": 0.727273 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.68346, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.68346, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 3312, + "balancedV1": 3512, + "qarinahV2": 3312, + "qarinahV2NoGraph": 3312, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.458466, + "directRecallAt10": 1, + "supportingRecallAt10": 0.090909 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 952, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.518493, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 915, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.315059, + "directRecallAt10": 0, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2": { + "deliveredTokens": 952, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.518493, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 952, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.518493, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.631297, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1928, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.58431, + "directRecallAt10": 1, + "supportingRecallAt10": 0.363636 + } + }, + "balancedV1": { + "deliveredTokens": 1962, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 1928, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.58431, + "directRecallAt10": 1, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1928, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.58431, + "directRecallAt10": 1, + "supportingRecallAt10": 0.363636 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.756234, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3926, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.68346, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + } + }, + "balancedV1": { + "deliveredTokens": 3887, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.727273, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.640479, + "directRecallAt10": 1, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 3926, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.68346, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3926, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.68346, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.68346, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.727273, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.640479, + "directRecallAt10": 1, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.68346, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.68346, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.507163, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-14608", + "repositorySequence": 70, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.30103, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3367, + "balancedV1": 3512, + "qarinahV2": 3367, + "qarinahV2NoGraph": 3367, + "oracle": 515 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 437, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 437, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 437, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 0, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 952, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 952, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 952, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1998, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1972, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.315465, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1998, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1998, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3994, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3938, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.30103, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3994, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3994, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.30103, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.219231, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "NO_CODE_ENTITY_MATCH", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-14667", + "repositorySequence": 71, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.784982, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + }, + "balancedV1": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.636364, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.793584, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.784982, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.784982, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 3458, + "balancedV1": 3538, + "qarinahV2": 3458, + "qarinahV2NoGraph": 3458, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 484, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 394, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 484, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 484, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 992, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 903, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2": { + "deliveredTokens": 992, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 992, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1979, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "balancedV1": { + "deliveredTokens": 1941, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.545455, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2": { + "deliveredTokens": 1979, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1979, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3858, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.784982, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "balancedV1": { + "deliveredTokens": 3997, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.636364, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.793584, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 3858, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.784982, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3858, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.784982, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.784982, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.636364, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.793584, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.784982, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.784982, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.43, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-14672", + "repositorySequence": 72, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 10, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.48475, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.727273, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.570765, + "directRecallAt10": 1, + "supportingRecallAt10": 0.727273 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.48475, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.48475, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 4631, + "balancedV1": 3953, + "qarinahV2": 4631, + "qarinahV2NoGraph": 4631, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 500, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 400, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 500, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 500, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 901, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 997, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.384773, + "directRecallAt10": 0, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2": { + "deliveredTokens": 901, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 901, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 982, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.631297, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1967, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.162237, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 1871, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.238648, + "directRecallAt10": 0, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2": { + "deliveredTokens": 1967, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.162237, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1967, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.162237, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.756234, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.48475, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + } + }, + "balancedV1": { + "deliveredTokens": 3953, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.727273, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.570765, + "directRecallAt10": 1, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 3984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.48475, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.48475, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.48475, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.727273, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.570765, + "directRecallAt10": 1, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.48475, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.48475, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.489633, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-14730", + "repositorySequence": 73, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.636364, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.796761, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + }, + "balancedV1": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.781805, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.636364, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.796761, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.636364, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.796761, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 3518, + "balancedV1": 3445, + "qarinahV2": 3518, + "qarinahV2NoGraph": 3518, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 394, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 394, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 394, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 394, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 909, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2": { + "deliveredTokens": 909, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 909, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1919, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "balancedV1": { + "deliveredTokens": 1916, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.636364, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.796761, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 1919, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1919, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3900, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.636364, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.796761, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "balancedV1": { + "deliveredTokens": 3988, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.781805, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 3900, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.636364, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.796761, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3900, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.636364, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.796761, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.636364, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.796761, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.781805, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.636364, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.796761, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.636364, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.796761, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.2975, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-14752", + "repositorySequence": 74, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.264068, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.264068, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.264068, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4799, + "balancedV1": 3292, + "qarinahV2": 4799, + "qarinahV2NoGraph": 4799, + "oracle": 626 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 449, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 251, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 990, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 915, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 990, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 990, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1926, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.264068, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 1920, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1926, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.264068, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1926, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.264068, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3937, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.264068, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 3966, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3937, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.264068, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3937, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.264068, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.264068, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.264068, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.264068, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.350615, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-14787", + "repositorySequence": 75, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.202107, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.452508, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.202107, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.202107, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4580, + "balancedV1": 3445, + "qarinahV2": 4580, + "qarinahV2NoGraph": 4580, + "oracle": 1678 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 424, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 444, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 941, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 941, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 941, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 444, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1879, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1920, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 1879, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1879, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 1678, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3964, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.202107, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 3950, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.452508, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3964, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.202107, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3964, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.202107, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 1678, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.202107, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.452508, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.202107, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.202107, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 1678, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.355882, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "NO_CODE_ENTITY_MATCH", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-14855", + "repositorySequence": 76, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.204382, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.204382, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.204382, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4590, + "balancedV1": 3600, + "qarinahV2": 4590, + "qarinahV2NoGraph": 4590, + "oracle": 626 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 251, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 952, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 952, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 952, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1891, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1962, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1891, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1891, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.204382, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 3906, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.204382, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.204382, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.204382, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.204382, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.204382, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.425, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-14915", + "repositorySequence": 77, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.260648, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.260648, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.260648, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4489, + "balancedV1": 3456, + "qarinahV2": 4489, + "qarinahV2NoGraph": 4489, + "oracle": 1129 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 424, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 0, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 954, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 954, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 954, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 614, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.826235, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1941, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1953, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1941, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1941, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 1129, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3927, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.260648, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 3984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3927, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.260648, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3927, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.260648, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 1129, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.260648, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.260648, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.260648, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 1129, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.421429, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-14997", + "repositorySequence": 78, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 10, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.399887, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.727273, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.482414, + "directRecallAt10": 0, + "supportingRecallAt10": 0.727273 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.399887, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.399887, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 4783, + "balancedV1": 3701, + "qarinahV2": 4783, + "qarinahV2NoGraph": 4783, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 970, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 915, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.325653, + "directRecallAt10": 0, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2": { + "deliveredTokens": 970, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 970, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 982, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.631297, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.09642, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 1920, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.238648, + "directRecallAt10": 0, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2": { + "deliveredTokens": 1971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.09642, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.09642, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.756234, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3907, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.221356, + "directRecallAt10": 0, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 3991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.727273, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.482414, + "directRecallAt10": 0, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 3907, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.221356, + "directRecallAt10": 0, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3907, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.221356, + "directRecallAt10": 0, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.399887, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.727273, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.482414, + "directRecallAt10": 0, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.399887, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.399887, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.392656, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-14999", + "repositorySequence": 79, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.454545, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.514186, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + }, + "balancedV1": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.705905, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.454545, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.514186, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.454545, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.514186, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 3115, + "balancedV1": 3299, + "qarinahV2": 3115, + "qarinahV2NoGraph": 3115, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 394, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 970, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2": { + "deliveredTokens": 970, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 970, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1998, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.31488, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 1953, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.554143, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2": { + "deliveredTokens": 1998, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.31488, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1998, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.31488, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3982, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.454545, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.514186, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "balancedV1": { + "deliveredTokens": 3921, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.705905, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2": { + "deliveredTokens": 3982, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.454545, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.514186, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3982, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.454545, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.514186, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.454545, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.514186, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.705905, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.454545, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.514186, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.454545, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.514186, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.411842, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "SINGLE_OR_NO_SUPPORT", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15061", + "repositorySequence": 80, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4435, + "balancedV1": 3296, + "qarinahV2": 4435, + "qarinahV2NoGraph": 4435, + "oracle": 515 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 426, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 416, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 426, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 426, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 0, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 943, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 973, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 943, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 943, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1998, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1912, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1998, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1998, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3976, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 3984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3976, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3976, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.524138, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15202", + "repositorySequence": 81, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4393, + "balancedV1": 3798, + "qarinahV2": 4393, + "qarinahV2NoGraph": 4393, + "oracle": 523 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 470, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 470, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 470, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 233, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 885, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 885, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 885, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1930, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1871, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1930, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1930, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3923, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 3929, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3923, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3923, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.493357, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15213", + "repositorySequence": 82, + "positiveUnderStructuralOracle": true, + "directRecords": 4, + "supportingRecords": 7, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.545455, + "precisionAt5": 0.4, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.42679, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.545455 + }, + "balancedV1": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.690032, + "directRecallAt10": 1, + "supportingRecallAt10": 0.636364 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.545455, + "precisionAt5": 0.4, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.42679, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.545455 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.545455, + "precisionAt5": 0.4, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.42679, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.545455 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 3533, + "balancedV1": 3820, + "qarinahV2": 3533, + "qarinahV2NoGraph": 3533, + "oracle": 3195 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 480, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.103447, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 404, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.168715, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 480, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.103447, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 480, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.103447, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.506145, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 999, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.323886, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 991, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.577599, + "directRecallAt10": 0.75, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2": { + "deliveredTokens": 999, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.323886, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 999, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.323886, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 955, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.834991, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1921, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.272727, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.205564, + "directRecallAt10": 0, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 1972, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.671536, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2": { + "deliveredTokens": 1921, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.272727, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.205564, + "directRecallAt10": 0, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1921, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.272727, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.205564, + "directRecallAt10": 0, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 1933, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.727273, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.938956, + "directRecallAt10": 1, + "supportingRecallAt10": 0.727273 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3992, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.545455, + "precisionAt5": 0.4, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.42679, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.545455 + } + }, + "balancedV1": { + "deliveredTokens": 3951, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.690032, + "directRecallAt10": 1, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 3992, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.545455, + "precisionAt5": 0.4, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.42679, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3992, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.545455, + "precisionAt5": 0.4, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.42679, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.545455 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.545455, + "precisionAt5": 0.4, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.42679, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.545455 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.690032, + "directRecallAt10": 1, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.545455, + "precisionAt5": 0.4, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.42679, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.545455, + "precisionAt5": 0.4, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.42679, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.545455 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.503846, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15252", + "repositorySequence": 83, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.602052, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + }, + "balancedV1": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.727273, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.795166, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.602052, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.602052, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 3622, + "balancedV1": 3953, + "qarinahV2": 3622, + "qarinahV2NoGraph": 3622, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 881, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 997, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.538886, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2": { + "deliveredTokens": 881, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 881, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1954, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 1962, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 1954, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1954, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3932, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.602052, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "balancedV1": { + "deliveredTokens": 3953, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.727273, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.795166, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 3932, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.602052, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3932, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.602052, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.602052, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.727273, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.795166, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.602052, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.602052, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.45079, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15320", + "repositorySequence": 84, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.596538, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + }, + "balancedV1": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.768227, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.596538, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.596538, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 3609, + "balancedV1": 3600, + "qarinahV2": 3609, + "qarinahV2NoGraph": 3609, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 480, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 484, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 480, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 480, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 992, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 903, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2": { + "deliveredTokens": 992, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 992, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1905, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.453743, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 1875, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2": { + "deliveredTokens": 1905, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.453743, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1905, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.453743, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3984, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.596538, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "balancedV1": { + "deliveredTokens": 3890, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.768227, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 3984, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.596538, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3984, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.596538, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.596538, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.768227, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.596538, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.596538, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.537023, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15347", + "repositorySequence": 85, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.177239, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.177239, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.177239, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3739, + "balancedV1": 3292, + "qarinahV2": 3739, + "qarinahV2NoGraph": 3739, + "oracle": 626 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 437, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 437, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 437, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 251, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 965, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 965, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 965, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1947, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1871, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1947, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1947, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3954, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.177239, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 3906, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3954, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.177239, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3954, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.177239, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.177239, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.177239, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.177239, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.424982, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15388", + "repositorySequence": 86, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 4655, + "balancedV1": 3383, + "qarinahV2": 4655, + "qarinahV2NoGraph": 4655, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 491, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 491, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 491, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 491, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 993, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 997, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 993, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 993, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1928, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1962, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1928, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1928, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3981, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3889, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3981, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3981, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.64403, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15400", + "repositorySequence": 87, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.25, + "ndcgAt10": 0.358534, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "balancedV1": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.703918, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.25, + "ndcgAt10": 0.358534, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.25, + "ndcgAt10": 0.358534, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4577, + "balancedV1": 3611, + "qarinahV2": 4577, + "qarinahV2NoGraph": 4577, + "oracle": 1678 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 466, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 444, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 920, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 915, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 920, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 920, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 444, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1987, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.202107, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 1962, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.703918, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 1987, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.202107, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1987, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.202107, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 1678, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3929, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.25, + "ndcgAt10": 0.358534, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 3906, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.703918, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3929, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.25, + "ndcgAt10": 0.358534, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3929, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.25, + "ndcgAt10": 0.358534, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 1678, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.25, + "ndcgAt10": 0.358534, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.703918, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.25, + "ndcgAt10": 0.358534, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.25, + "ndcgAt10": 0.358534, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 1678, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.4075, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15498", + "repositorySequence": 88, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4171, + "balancedV1": 3234, + "qarinahV2": 4171, + "qarinahV2NoGraph": 4171, + "oracle": 295 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 449, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 295, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 295, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1882, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1978, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1882, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1882, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 295, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3996, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 3931, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3996, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3996, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 295, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 295, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.523333, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15695", + "repositorySequence": 89, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.675416, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + }, + "balancedV1": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.727273, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.775994, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.675416, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.675416, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 4664, + "balancedV1": 3701, + "qarinahV2": 4664, + "qarinahV2NoGraph": 4664, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 952, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 915, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2": { + "deliveredTokens": 952, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 952, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1992, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 1920, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.424926, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2": { + "deliveredTokens": 1992, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1992, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3945, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.678049, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "balancedV1": { + "deliveredTokens": 3934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.727273, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.775994, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 3945, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.678049, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3945, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.678049, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.675416, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.727273, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.775994, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.675416, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.675416, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.436029, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15738", + "repositorySequence": 90, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 10, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.578377, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + }, + "balancedV1": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.636364, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.506124, + "directRecallAt10": 0, + "supportingRecallAt10": 0.636364 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.578377, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.578377, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 3684, + "balancedV1": 3525, + "qarinahV2": 3684, + "qarinahV2NoGraph": 3684, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.09642, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.09642, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.09642, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.28926, + "directRecallAt10": 1, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 997, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.374178, + "directRecallAt10": 0, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2": { + "deliveredTokens": 970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.28926, + "directRecallAt10": 1, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.28926, + "directRecallAt10": 1, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 992, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.697114, + "directRecallAt10": 1, + "supportingRecallAt10": 0.363636 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1960, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.530168, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "balancedV1": { + "deliveredTokens": 1962, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 1960, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.530168, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1960, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.530168, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.756234, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3979, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.578377, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + } + }, + "balancedV1": { + "deliveredTokens": 3887, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.636364, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.506124, + "directRecallAt10": 0, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 3979, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.578377, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3979, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.578377, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.578377, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.636364, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.506124, + "directRecallAt10": 0, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.578377, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.578377, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.502955, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15781", + "repositorySequence": 91, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.184576, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.177239, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.184576, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.184576, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4098, + "balancedV1": 3572, + "qarinahV2": 4098, + "qarinahV2NoGraph": 4098, + "oracle": 523 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 233, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 915, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1968, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1962, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1968, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1968, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 4000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.184576, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 3993, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.177239, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 4000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.184576, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 4000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.184576, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.184576, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.177239, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.184576, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.184576, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "DIRECTLY_SUPPORTED", + "decision": "ACCEPT_DIRECT", + "score": 0.653846, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "CONSERVATIVE_DIRECT_THRESHOLD_MET" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15789", + "repositorySequence": 92, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.391066, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4150, + "balancedV1": 3764, + "qarinahV2": 4150, + "qarinahV2NoGraph": 4150, + "oracle": 1678 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 400, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 444, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 968, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 968, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 968, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 444, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1955, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 1881, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.234639, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 1955, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1955, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 1678, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3986, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 3979, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.391066, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3986, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3986, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 1678, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.391066, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 1678, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.553409, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15790", + "repositorySequence": 93, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.571429, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.184576, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.571429, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.571429, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3933, + "balancedV1": 3234, + "qarinahV2": 3933, + "qarinahV2NoGraph": 3933, + "oracle": 523 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 444, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 444, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 444, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 233, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 987, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 987, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 987, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1911, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 1995, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1911, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1911, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3933, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.571429, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3966, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.184576, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 3933, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.571429, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3933, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.571429, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.571429, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.184576, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.571429, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.571429, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.395161, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "NO_CODE_ENTITY_MATCH", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15814", + "repositorySequence": 94, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.570859, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.727273, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.68512, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.570859, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.570859, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 4699, + "balancedV1": 3953, + "qarinahV2": 4699, + "qarinahV2NoGraph": 4699, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 952, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 975, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.453743, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2": { + "deliveredTokens": 952, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 952, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1974, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.343697, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 1962, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.248908, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 1974, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.343697, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1974, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.343697, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3939, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.363636, + "recallAt10": 0.454545, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.507238, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "balancedV1": { + "deliveredTokens": 3953, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.727273, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.68512, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 3939, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.363636, + "recallAt10": 0.454545, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.507238, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3939, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.363636, + "recallAt10": 0.454545, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.507238, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.570859, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.727273, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.68512, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.570859, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.570859, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.50145, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15819", + "repositorySequence": 95, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.797723, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.184576, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "qarinahV2": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.797723, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.797723, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2896, + "balancedV1": 3538, + "qarinahV2": 2896, + "qarinahV2NoGraph": 2896, + "oracle": 523 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 470, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 470, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 470, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 233, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 985, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 985, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 985, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1962, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.193426, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 1962, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1962, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3906, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.797723, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3979, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.184576, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 3906, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.797723, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3906, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.797723, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.797723, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.184576, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.797723, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.797723, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.337805, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "NO_CODE_ENTITY_MATCH", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15851", + "repositorySequence": 96, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.506784, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + }, + "balancedV1": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.727273, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.751092, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.506784, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.506784, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 4878, + "balancedV1": 3953, + "qarinahV2": 4878, + "qarinahV2NoGraph": 4878, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 510, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 466, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2": { + "deliveredTokens": 510, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 510, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 909, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 975, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.424926, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2": { + "deliveredTokens": 909, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 909, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1981, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.538886, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "balancedV1": { + "deliveredTokens": 1962, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.31488, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 1981, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.538886, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1981, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.538886, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3964, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.506784, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "balancedV1": { + "deliveredTokens": 3953, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.727273, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.751092, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 3964, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.506784, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3964, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.506784, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.506784, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.727273, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.751092, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.506784, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.506784, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "DIRECTLY_SUPPORTED", + "decision": "ACCEPT_DIRECT", + "score": 0.671875, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "CONSERVATIVE_DIRECT_THRESHOLD_MET" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15902", + "repositorySequence": 97, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.289065, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4365, + "balancedV1": 3516, + "qarinahV2": 4365, + "qarinahV2NoGraph": 4365, + "oracle": 515 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 484, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 484, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 484, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 0, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 999, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 885, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 999, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 999, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1982, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1962, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1982, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1982, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3990, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 4000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.289065, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3990, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3990, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.289065, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.375, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "NO_CODE_ENTITY_MATCH", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15996", + "repositorySequence": 98, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 10, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.454545, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.704438, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + }, + "balancedV1": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.628633, + "directRecallAt10": 1, + "supportingRecallAt10": 0.636364 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.454545, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.704438, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.454545, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.704438, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 3834, + "balancedV1": 3456, + "qarinahV2": 3834, + "qarinahV2NoGraph": 3834, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.458466, + "directRecallAt10": 1, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.458466, + "directRecallAt10": 1, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.458466, + "directRecallAt10": 1, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.458466, + "directRecallAt10": 1, + "supportingRecallAt10": 0.090909 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.458466, + "directRecallAt10": 1, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.450589, + "directRecallAt10": 0, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.458466, + "directRecallAt10": 1, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.458466, + "directRecallAt10": 1, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.631297, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1936, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 1916, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.641477, + "directRecallAt10": 1, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 1936, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1936, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.756234, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3972, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.454545, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.704438, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "balancedV1": { + "deliveredTokens": 3999, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.628633, + "directRecallAt10": 1, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 3972, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.454545, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.704438, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3972, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.454545, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.704438, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.454545, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.704438, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.628633, + "directRecallAt10": 1, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.454545, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.704438, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.454545, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.704438, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.489858, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-16041", + "repositorySequence": 99, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3650, + "balancedV1": 3634, + "qarinahV2": 3650, + "qarinahV2NoGraph": 3650, + "oracle": 515 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 0, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 943, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 933, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 943, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 943, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1980, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3887, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 3940, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3887, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3887, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.389286, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-16046", + "repositorySequence": 100, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.148041, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.4018, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.148041, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.148041, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4652, + "balancedV1": 3307, + "qarinahV2": 4652, + "qarinahV2NoGraph": 4652, + "oracle": 1678 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 505, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 505, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 505, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 444, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 890, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 890, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 890, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 444, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1999, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1995, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.416181, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 1999, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1999, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 1678, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3964, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 3878, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.4018, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3964, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3964, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 1678, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.148041, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.4018, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.148041, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.148041, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 1678, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.535, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-16139", + "repositorySequence": 101, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.280944, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.218617, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.280944, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.280944, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4860, + "balancedV1": 3611, + "qarinahV2": 4860, + "qarinahV2NoGraph": 4860, + "oracle": 1240 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 251, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.242076, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 865, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1928, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1962, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1928, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1928, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 1240, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3907, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.280944, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 3906, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.218617, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 3907, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.280944, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3907, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.280944, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 1240, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.280944, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.218617, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.280944, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.280944, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 1240, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.381613, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-16229", + "repositorySequence": 102, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.333333, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.333333, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.333333, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3827, + "balancedV1": 3798, + "qarinahV2": 3827, + "qarinahV2NoGraph": 3827, + "oracle": 515 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 500, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 500, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 500, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 0, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 910, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 910, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 910, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1916, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.333333, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3929, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.333333, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.333333, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.333333, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.333333, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.333333, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.4625, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-16255", + "repositorySequence": 103, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.204382, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.204382, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.204382, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4539, + "balancedV1": 3634, + "qarinahV2": 4539, + "qarinahV2NoGraph": 4539, + "oracle": 626 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 510, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 510, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 510, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 251, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 943, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 915, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 943, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 943, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1882, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1920, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1882, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1882, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3931, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.204382, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 3940, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3931, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.204382, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3931, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.204382, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.204382, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.204382, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.204382, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.465357, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-16379", + "repositorySequence": 104, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4171, + "balancedV1": 3292, + "qarinahV2": 4171, + "qarinahV2NoGraph": 4171, + "oracle": 523 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 444, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 498, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 444, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 444, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 233, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 891, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 973, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 891, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 891, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1973, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1973, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1973, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3909, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 3892, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3909, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3909, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.557703, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-16400", + "repositorySequence": 105, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.184576, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.184576, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.184576, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4740, + "balancedV1": 3764, + "qarinahV2": 4740, + "qarinahV2NoGraph": 4740, + "oracle": 626 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 251, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 915, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1962, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3870, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.193426, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 3997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3870, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.193426, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3870, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.193426, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.184576, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.184576, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.184576, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.388, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-16408", + "repositorySequence": 106, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 10, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.606511, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.727273, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.553474, + "directRecallAt10": 1, + "supportingRecallAt10": 0.727273 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.606511, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.606511, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 4754, + "balancedV1": 3953, + "qarinahV2": 4754, + "qarinahV2NoGraph": 4754, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 476, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 954, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.35417, + "directRecallAt10": 0, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2": { + "deliveredTokens": 954, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 954, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 982, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.631297, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1968, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.58431, + "directRecallAt10": 1, + "supportingRecallAt10": 0.363636 + } + }, + "balancedV1": { + "deliveredTokens": 1998, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.221356, + "directRecallAt10": 0, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2": { + "deliveredTokens": 1968, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.58431, + "directRecallAt10": 1, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1968, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.58431, + "directRecallAt10": 1, + "supportingRecallAt10": 0.363636 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.756234, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3929, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.610545, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "balancedV1": { + "deliveredTokens": 3953, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.727273, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.553474, + "directRecallAt10": 1, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 3929, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.610545, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3929, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.610545, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.606511, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.727273, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.553474, + "directRecallAt10": 1, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.606511, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.454545, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.606511, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.579221, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-16527", + "repositorySequence": 107, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.524981, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.184576, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.524981, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.524981, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4355, + "balancedV1": 3372, + "qarinahV2": 4355, + "qarinahV2NoGraph": 4355, + "oracle": 626 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 463, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 386, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 463, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 463, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 251, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 920, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 948, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 920, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 920, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1932, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.543771, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1995, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1932, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.543771, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1932, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.543771, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3983, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.524981, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3913, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.184576, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 3983, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.524981, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3983, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.524981, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.524981, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.184576, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.524981, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.524981, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.5, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-16595", + "repositorySequence": 108, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.826333, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + }, + "balancedV1": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.836459, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.826333, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.826333, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 3577, + "balancedV1": 3512, + "qarinahV2": 3577, + "qarinahV2NoGraph": 3577, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 500, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 500, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 500, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 959, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2": { + "deliveredTokens": 959, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 959, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1940, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.690648, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "balancedV1": { + "deliveredTokens": 1950, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.851236, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 1940, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.690648, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1940, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.545455, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.690648, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3998, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.826333, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "balancedV1": { + "deliveredTokens": 3938, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.836459, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 3998, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.826333, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3998, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.826333, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.826333, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.836459, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.826333, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.826333, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.523462, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-16816", + "repositorySequence": 109, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4423, + "balancedV1": 3634, + "qarinahV2": 4423, + "qarinahV2NoGraph": 4423, + "oracle": 626 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 510, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 419, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 510, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 510, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 251, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 973, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 885, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 973, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 973, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1992, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1890, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1992, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1992, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3940, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 3940, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3940, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3940, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.377174, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-16820", + "repositorySequence": 110, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.672239, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + }, + "balancedV1": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.636364, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.728912, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.672239, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.672239, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 3260, + "balancedV1": 3132, + "qarinahV2": 3260, + "qarinahV2NoGraph": 3260, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 475, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 424, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 475, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 475, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "balancedV1": { + "deliveredTokens": 915, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.453743, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2": { + "deliveredTokens": 934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1989, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.363636, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.542364, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "balancedV1": { + "deliveredTokens": 1887, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 1989, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.363636, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.542364, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1989, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.363636, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.542364, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3952, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.672239, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "balancedV1": { + "deliveredTokens": 3797, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.636364, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.728912, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 3952, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.672239, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3952, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.672239, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.672239, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.636364, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.728912, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.672239, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.672239, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.34375, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "NO_CODE_ENTITY_MATCH", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-16873", + "repositorySequence": 111, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 4560, + "balancedV1": 3953, + "qarinahV2": 4560, + "qarinahV2NoGraph": 4560, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 459, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 499, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 459, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 459, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 910, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 915, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 910, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 910, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1882, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1962, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1882, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1882, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3959, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3953, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3959, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3959, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.439009, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-16910", + "repositorySequence": 112, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.636364, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.790951, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + }, + "balancedV1": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.727273, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.857205, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.636364, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.790951, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.636364, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.790951, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 3199, + "balancedV1": 3512, + "qarinahV2": 3199, + "qarinahV2NoGraph": 3199, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 500, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 419, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 500, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 500, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 904, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "balancedV1": { + "deliveredTokens": 903, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2": { + "deliveredTokens": 904, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 904, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1977, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.545455, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "balancedV1": { + "deliveredTokens": 1950, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.727273, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.863015, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 1977, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.545455, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1977, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.545455, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3985, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.636364, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.790951, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "balancedV1": { + "deliveredTokens": 3933, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.727273, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.857205, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 3985, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.636364, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.790951, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3985, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.636364, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.790951, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.636364, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.790951, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.727273, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.857205, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.636364, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.790951, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.636364, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.790951, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.470862, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-17051", + "repositorySequence": 113, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.719118, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + }, + "balancedV1": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.777872, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.719118, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.719118, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 3313, + "balancedV1": 3600, + "qarinahV2": 3313, + "qarinahV2NoGraph": 3313, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 385, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 909, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 991, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2": { + "deliveredTokens": 909, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 909, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1942, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.510069, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "balancedV1": { + "deliveredTokens": 1883, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.636364, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.78855, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 1942, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.510069, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1942, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.510069, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3929, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.719118, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "balancedV1": { + "deliveredTokens": 3910, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.777872, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 3929, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.719118, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3929, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.719118, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.719118, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.777872, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.719118, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.636364, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.719118, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.454268, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-17087", + "repositorySequence": 114, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 10, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.454545, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.583599, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + }, + "balancedV1": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.727273, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.588979, + "directRecallAt10": 0, + "supportingRecallAt10": 0.727273 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.454545, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.583599, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.454545, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.583599, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 3437, + "balancedV1": 3701, + "qarinahV2": 3437, + "qarinahV2NoGraph": 3437, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 500, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 449, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 500, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 500, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.458466, + "directRecallAt10": 1, + "supportingRecallAt10": 0.090909 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 959, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.518493, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.450589, + "directRecallAt10": 0, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2": { + "deliveredTokens": 959, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.518493, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 959, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.518493, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.631297, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1982, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.537595, + "directRecallAt10": 1, + "supportingRecallAt10": 0.363636 + } + }, + "balancedV1": { + "deliveredTokens": 1950, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.727273, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.595744, + "directRecallAt10": 0, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 1982, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.537595, + "directRecallAt10": 1, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1982, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.537595, + "directRecallAt10": 1, + "supportingRecallAt10": 0.363636 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.756234, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3969, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.454545, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.583599, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "balancedV1": { + "deliveredTokens": 3996, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.727273, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.588979, + "directRecallAt10": 0, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 3969, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.454545, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.583599, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3969, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.454545, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.583599, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.454545, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.583599, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.727273, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.588979, + "directRecallAt10": 0, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.454545, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.583599, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.454545, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.583599, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.494279, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "matplotlib/matplotlib", + "instanceId": "matplotlib__matplotlib-23476", + "repositorySequence": 6, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 4, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2763, + "balancedV1": 2763, + "qarinahV2": 2763, + "qarinahV2NoGraph": 2763, + "oracle": 2470 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 802, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 802, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 802, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1746, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 1944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2": { + "deliveredTokens": 1746, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1746, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 1961, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2470, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2470, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.46133, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 8, + "noTemporalReturnedItems": 10 + }, + { + "repository": "matplotlib/matplotlib", + "instanceId": "matplotlib__matplotlib-23562", + "repositorySequence": 7, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2763, + "balancedV1": 2763, + "qarinahV2": 2763, + "qarinahV2NoGraph": 2763, + "oracle": 293 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 293, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 293, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 293, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 293, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 819, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 819, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 819, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 293, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1763, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1944, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1763, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1763, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 293, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 293, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 293, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.482807, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "matplotlib/matplotlib", + "instanceId": "matplotlib__matplotlib-23563", + "repositorySequence": 8, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2763, + "balancedV1": 2763, + "qarinahV2": 2763, + "qarinahV2NoGraph": 2763, + "oracle": 293 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 293, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 293, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 293, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 293, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 293, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1746, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1944, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1746, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1746, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 293, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 293, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 293, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.465178, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "matplotlib/matplotlib", + "instanceId": "matplotlib__matplotlib-23913", + "repositorySequence": 9, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 4, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.904717, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.904717, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.904717, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2763, + "balancedV1": 2763, + "qarinahV2": 2763, + "qarinahV2NoGraph": 2763, + "oracle": 2470 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 784, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 784, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 784, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1819, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.753698, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "balancedV1": { + "deliveredTokens": 1819, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2": { + "deliveredTokens": 1819, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.753698, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1819, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.753698, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "oracle": { + "deliveredTokens": 1961, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.904717, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.904717, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.904717, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2470, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.904717, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.904717, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.904717, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2470, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.425893, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "SINGLE_OR_NO_SUPPORT", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "matplotlib/matplotlib", + "instanceId": "matplotlib__matplotlib-23964", + "repositorySequence": 10, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 4, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2763, + "balancedV1": 2763, + "qarinahV2": 2763, + "qarinahV2NoGraph": 2763, + "oracle": 2470 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1746, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 1944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2": { + "deliveredTokens": 1746, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1746, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 1961, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2470, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2470, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.456857, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "matplotlib/matplotlib", + "instanceId": "matplotlib__matplotlib-23987", + "repositorySequence": 11, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 4, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.76064, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.76064, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.76064, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2763, + "balancedV1": 2763, + "qarinahV2": 2763, + "qarinahV2NoGraph": 2763, + "oracle": 2470 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 293, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2": { + "deliveredTokens": 293, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 293, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 802, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.246302, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 802, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.246302, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 802, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.246302, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1746, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.441492, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 1944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2": { + "deliveredTokens": 1746, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.441492, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1746, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.441492, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 1961, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.76064, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.76064, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.76064, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2470, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.76064, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.76064, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.76064, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2470, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.598701, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "matplotlib/matplotlib", + "instanceId": "matplotlib__matplotlib-24149", + "repositorySequence": 12, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 4, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2763, + "balancedV1": 2763, + "qarinahV2": 2763, + "qarinahV2NoGraph": 2763, + "oracle": 2470 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "balancedV1": { + "deliveredTokens": 1944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2": { + "deliveredTokens": 1944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "oracle": { + "deliveredTokens": 1961, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2470, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2470, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.463077, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "matplotlib/matplotlib", + "instanceId": "matplotlib__matplotlib-24265", + "repositorySequence": 13, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.677664, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.838184, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.677664, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.677664, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2763, + "balancedV1": 2763, + "qarinahV2": 2763, + "qarinahV2NoGraph": 2763, + "oracle": 2470 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.219221, + "directRecallAt10": 0, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.219221, + "directRecallAt10": 0, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.219221, + "directRecallAt10": 0, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.219221, + "directRecallAt10": 0, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.657663, + "directRecallAt10": 1, + "supportingRecallAt10": 0.25 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 802, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.219221, + "directRecallAt10": 0, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.63416, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 802, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.219221, + "directRecallAt10": 0, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 802, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.219221, + "directRecallAt10": 0, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.795976, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1746, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.328832, + "directRecallAt10": 0, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 1944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.743771, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2": { + "deliveredTokens": 1746, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.328832, + "directRecallAt10": 0, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1746, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.328832, + "directRecallAt10": 0, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 1961, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.905587, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.677664, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.838184, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.677664, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.677664, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2470, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.677664, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.838184, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.677664, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.677664, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2470, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.595652, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "matplotlib/matplotlib", + "instanceId": "matplotlib__matplotlib-24334", + "repositorySequence": 14, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 4, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2763, + "balancedV1": 2763, + "qarinahV2": 2763, + "qarinahV2NoGraph": 2763, + "oracle": 2470 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1819, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "balancedV1": { + "deliveredTokens": 1819, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2": { + "deliveredTokens": 1819, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1819, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "oracle": { + "deliveredTokens": 1961, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2470, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2470, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.423182, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "matplotlib/matplotlib", + "instanceId": "matplotlib__matplotlib-24970", + "repositorySequence": 15, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 4, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.904717, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.904717, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.904717, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2763, + "balancedV1": 2763, + "qarinahV2": 2763, + "qarinahV2NoGraph": 2763, + "oracle": 2470 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 293, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2": { + "deliveredTokens": 293, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 293, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1746, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.58557, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 1944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2": { + "deliveredTokens": 1746, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.58557, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1746, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.58557, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 1961, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.904717, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.904717, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.904717, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2470, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.904717, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.904717, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.904717, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2470, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.582278, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "matplotlib/matplotlib", + "instanceId": "matplotlib__matplotlib-25079", + "repositorySequence": 16, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 4, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2763, + "balancedV1": 2763, + "qarinahV2": 2763, + "qarinahV2NoGraph": 2763, + "oracle": 2470 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1979, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "balancedV1": { + "deliveredTokens": 1944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2": { + "deliveredTokens": 1979, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1979, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "oracle": { + "deliveredTokens": 1961, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2470, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2470, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.531818, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "matplotlib/matplotlib", + "instanceId": "matplotlib__matplotlib-25311", + "repositorySequence": 17, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 4, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2763, + "balancedV1": 2763, + "qarinahV2": 2763, + "qarinahV2NoGraph": 2763, + "oracle": 2470 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1746, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 1944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2": { + "deliveredTokens": 1746, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1746, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 1961, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2470, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2470, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.604762, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "matplotlib/matplotlib", + "instanceId": "matplotlib__matplotlib-25332", + "repositorySequence": 18, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 4, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.904717, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.904717, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.904717, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2763, + "balancedV1": 2763, + "qarinahV2": 2763, + "qarinahV2NoGraph": 2763, + "oracle": 2470 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 802, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 802, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 802, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1746, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.58557, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 1944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2": { + "deliveredTokens": 1746, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.58557, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1746, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.58557, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 1961, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.904717, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.904717, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.904717, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2470, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.904717, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.904717, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.904717, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2470, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.546451, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "matplotlib/matplotlib", + "instanceId": "matplotlib__matplotlib-25433", + "repositorySequence": 19, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 4, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2763, + "balancedV1": 2763, + "qarinahV2": 2763, + "qarinahV2NoGraph": 2763, + "oracle": 2470 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1746, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 1944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2": { + "deliveredTokens": 1746, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1746, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 1961, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2470, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2470, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.614205, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "matplotlib/matplotlib", + "instanceId": "matplotlib__matplotlib-25442", + "repositorySequence": 20, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 4, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2763, + "balancedV1": 2763, + "qarinahV2": 2763, + "qarinahV2NoGraph": 2763, + "oracle": 2470 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 802, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 802, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 802, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1746, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 1944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2": { + "deliveredTokens": 1746, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1746, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 1961, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2470, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2470, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.54912, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "matplotlib/matplotlib", + "instanceId": "matplotlib__matplotlib-25498", + "repositorySequence": 21, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 4, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2763, + "balancedV1": 2763, + "qarinahV2": 2763, + "qarinahV2NoGraph": 2763, + "oracle": 2470 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1979, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "balancedV1": { + "deliveredTokens": 1944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2": { + "deliveredTokens": 1979, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1979, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "oracle": { + "deliveredTokens": 1961, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2470, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2470, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.563333, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "matplotlib/matplotlib", + "instanceId": "matplotlib__matplotlib-26011", + "repositorySequence": 22, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 4, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2763, + "balancedV1": 2763, + "qarinahV2": 2763, + "qarinahV2NoGraph": 2763, + "oracle": 2470 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "balancedV1": { + "deliveredTokens": 1944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2": { + "deliveredTokens": 1944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "oracle": { + "deliveredTokens": 1961, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2470, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.982892, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2470, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.418609, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "matplotlib/matplotlib", + "instanceId": "matplotlib__matplotlib-26020", + "repositorySequence": 23, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2763, + "balancedV1": 2763, + "qarinahV2": 2763, + "qarinahV2NoGraph": 2763, + "oracle": 293 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 293, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 293, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1944, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1944, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1944, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1944, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 293, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 293, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 293, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.36946, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "mwaskom/seaborn", + "instanceId": "mwaskom__seaborn-3010", + "repositorySequence": 2, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 302, + "balancedV1": 302, + "qarinahV2": 302, + "qarinahV2NoGraph": 302, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 302, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 302, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 302, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 302, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 302, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 302, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 302, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 302, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 302, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 302, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 302, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 302, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 302, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 302, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 302, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 302, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 302, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 302, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 302, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 302, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.234877, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 4 + }, + { + "repository": "mwaskom/seaborn", + "instanceId": "mwaskom__seaborn-3190", + "repositorySequence": 3, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 302, + "balancedV1": 302, + "qarinahV2": 302, + "qarinahV2NoGraph": 302, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 302, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 302, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 302, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 302, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 302, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 302, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 302, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 302, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 302, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 302, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 302, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 302, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 302, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 302, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 302, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 302, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 302, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 302, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 302, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 302, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.284677, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 4 + }, + { + "repository": "mwaskom/seaborn", + "instanceId": "mwaskom__seaborn-3407", + "repositorySequence": 4, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 302, + "balancedV1": 302, + "qarinahV2": 302, + "qarinahV2NoGraph": 302, + "oracle": 302 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 302, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 302, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 302, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 302, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 302, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 302, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 302, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 302, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 302, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 302, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 302, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 302, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 302, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 302, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 302, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 302, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 302, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 302, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 302, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 302, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 302, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 302, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 302, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 302, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 302, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.318065, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 4 + }, + { + "repository": "pallets/flask", + "instanceId": "pallets__flask-4992", + "repositorySequence": 2, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 113, + "balancedV1": 113, + "qarinahV2": 113, + "qarinahV2NoGraph": 113, + "oracle": 113 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.260396, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 3 + }, + { + "repository": "pallets/flask", + "instanceId": "pallets__flask-5063", + "repositorySequence": 3, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 113, + "balancedV1": 113, + "qarinahV2": 113, + "qarinahV2NoGraph": 113, + "oracle": 113 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.20625, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "NO_CODE_ENTITY_MATCH", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 3 + }, + { + "repository": "psf/requests", + "instanceId": "psf__requests-1963", + "repositorySequence": 2, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 253, + "balancedV1": 253, + "qarinahV2": 253, + "qarinahV2NoGraph": 253, + "oracle": 253 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.306579, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 6 + }, + { + "repository": "psf/requests", + "instanceId": "psf__requests-2148", + "repositorySequence": 3, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 253, + "balancedV1": 253, + "qarinahV2": 253, + "qarinahV2NoGraph": 253, + "oracle": 253 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.246145, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 6 + }, + { + "repository": "psf/requests", + "instanceId": "psf__requests-2317", + "repositorySequence": 4, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 253, + "balancedV1": 253, + "qarinahV2": 253, + "qarinahV2NoGraph": 253, + "oracle": 253 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.405714, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "SINGLE_OR_NO_SUPPORT", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 6 + }, + { + "repository": "psf/requests", + "instanceId": "psf__requests-2674", + "repositorySequence": 5, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 253, + "balancedV1": 253, + "qarinahV2": 253, + "qarinahV2NoGraph": 253, + "oracle": 253 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.347297, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 6 + }, + { + "repository": "psf/requests", + "instanceId": "psf__requests-3362", + "repositorySequence": 6, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 253, + "balancedV1": 253, + "qarinahV2": 253, + "qarinahV2NoGraph": 253, + "oracle": 253 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.197872, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "NO_CODE_ENTITY_MATCH", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 6 + }, + { + "repository": "pydata/xarray", + "instanceId": "pydata__xarray-4094", + "repositorySequence": 2, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 215, + "balancedV1": 215, + "qarinahV2": 215, + "qarinahV2NoGraph": 215, + "oracle": 215 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.175403, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "NO_CODE_ENTITY_MATCH", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 5 + }, + { + "repository": "pydata/xarray", + "instanceId": "pydata__xarray-4248", + "repositorySequence": 3, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 215, + "balancedV1": 215, + "qarinahV2": 215, + "qarinahV2NoGraph": 215, + "oracle": 215 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.392188, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 5 + }, + { + "repository": "pydata/xarray", + "instanceId": "pydata__xarray-4493", + "repositorySequence": 4, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 215, + "balancedV1": 215, + "qarinahV2": 215, + "qarinahV2NoGraph": 215, + "oracle": 215 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.25, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 5 + }, + { + "repository": "pydata/xarray", + "instanceId": "pydata__xarray-5131", + "repositorySequence": 5, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 215, + "balancedV1": 215, + "qarinahV2": 215, + "qarinahV2NoGraph": 215, + "oracle": 215 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.179221, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "NO_CODE_ENTITY_MATCH", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 5 + }, + { + "repository": "pylint-dev/pylint", + "instanceId": "pylint-dev__pylint-6506", + "repositorySequence": 2, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 287, + "balancedV1": 287, + "qarinahV2": 287, + "qarinahV2NoGraph": 287, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 287, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 287, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 287, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 287, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 287, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 287, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 287, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 287, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 287, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 287, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.4218, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "SINGLE_OR_NO_SUPPORT", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 6 + }, + { + "repository": "pylint-dev/pylint", + "instanceId": "pylint-dev__pylint-7080", + "repositorySequence": 3, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 287, + "balancedV1": 287, + "qarinahV2": 287, + "qarinahV2NoGraph": 287, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 287, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 287, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 287, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 287, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 287, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 287, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 287, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 287, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 287, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 287, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.274436, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 6 + }, + { + "repository": "pylint-dev/pylint", + "instanceId": "pylint-dev__pylint-7114", + "repositorySequence": 4, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 287, + "balancedV1": 287, + "qarinahV2": 287, + "qarinahV2NoGraph": 287, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 287, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 287, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 287, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 287, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 287, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 287, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 287, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 287, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 287, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 287, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.362125, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 6 + }, + { + "repository": "pylint-dev/pylint", + "instanceId": "pylint-dev__pylint-7228", + "repositorySequence": 5, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 287, + "balancedV1": 287, + "qarinahV2": 287, + "qarinahV2NoGraph": 287, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 287, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 287, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 287, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 287, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 287, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 287, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 287, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 287, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 287, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 287, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.375337, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 6 + }, + { + "repository": "pylint-dev/pylint", + "instanceId": "pylint-dev__pylint-7993", + "repositorySequence": 6, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 287, + "balancedV1": 287, + "qarinahV2": 287, + "qarinahV2NoGraph": 287, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 287, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 287, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 287, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 287, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 287, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 287, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 287, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 287, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 287, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 287, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.440968, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "SINGLE_OR_NO_SUPPORT", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 6 + }, + { + "repository": "pytest-dev/pytest", + "instanceId": "pytest-dev__pytest-5413", + "repositorySequence": 4, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1044, + "balancedV1": 1044, + "qarinahV2": 1044, + "qarinahV2NoGraph": 1044, + "oracle": 1044 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.37875, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 9, + "noTemporalReturnedItems": 10 + }, + { + "repository": "pytest-dev/pytest", + "instanceId": "pytest-dev__pytest-5495", + "repositorySequence": 5, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1044, + "balancedV1": 1044, + "qarinahV2": 1044, + "qarinahV2NoGraph": 1044, + "oracle": 1044 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.397436, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 9, + "noTemporalReturnedItems": 10 + }, + { + "repository": "pytest-dev/pytest", + "instanceId": "pytest-dev__pytest-5692", + "repositorySequence": 6, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1044, + "balancedV1": 1044, + "qarinahV2": 1044, + "qarinahV2NoGraph": 1044, + "oracle": 1044 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.349569, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 8, + "noTemporalReturnedItems": 10 + }, + { + "repository": "pytest-dev/pytest", + "instanceId": "pytest-dev__pytest-6116", + "repositorySequence": 7, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1044, + "balancedV1": 1044, + "qarinahV2": 1044, + "qarinahV2NoGraph": 1044, + "oracle": 1044 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.214286, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "NO_CODE_ENTITY_MATCH", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "pytest-dev/pytest", + "instanceId": "pytest-dev__pytest-7168", + "repositorySequence": 8, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1044, + "balancedV1": 1044, + "qarinahV2": 1044, + "qarinahV2NoGraph": 1044, + "oracle": 1044 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.327971, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 8, + "noTemporalReturnedItems": 10 + }, + { + "repository": "pytest-dev/pytest", + "instanceId": "pytest-dev__pytest-7220", + "repositorySequence": 9, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1044, + "balancedV1": 1044, + "qarinahV2": 1044, + "qarinahV2NoGraph": 1044, + "oracle": 1044 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 738, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 738, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 738, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.416694, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "pytest-dev/pytest", + "instanceId": "pytest-dev__pytest-7373", + "repositorySequence": 10, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1044, + "balancedV1": 1044, + "qarinahV2": 1044, + "qarinahV2NoGraph": 1044, + "oracle": 1044 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.403247, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "pytest-dev/pytest", + "instanceId": "pytest-dev__pytest-7432", + "repositorySequence": 11, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1044, + "balancedV1": 1044, + "qarinahV2": 1044, + "qarinahV2NoGraph": 1044, + "oracle": 1044 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 738, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 738, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 738, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 738, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.426006, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "pytest-dev/pytest", + "instanceId": "pytest-dev__pytest-7490", + "repositorySequence": 12, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1044, + "balancedV1": 1044, + "qarinahV2": 1044, + "qarinahV2NoGraph": 1044, + "oracle": 1044 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.183982, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "NO_CODE_ENTITY_MATCH", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "pytest-dev/pytest", + "instanceId": "pytest-dev__pytest-8365", + "repositorySequence": 13, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1044, + "balancedV1": 1044, + "qarinahV2": 1044, + "qarinahV2NoGraph": 1044, + "oracle": 1044 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.3235, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "pytest-dev/pytest", + "instanceId": "pytest-dev__pytest-8906", + "repositorySequence": 14, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.757924, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.757924, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.757924, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.757924, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1044, + "balancedV1": 1044, + "qarinahV2": 1044, + "qarinahV2NoGraph": 1044, + "oracle": 1044 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.700276, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.700276, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.700276, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.700276, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.394809, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.394809, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.394809, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.394809, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.757924, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.757924, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.757924, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.757924, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.757924, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.757924, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.757924, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.757924, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.757924, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.757924, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.757924, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.757924, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.345031, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "pytest-dev/pytest", + "instanceId": "pytest-dev__pytest-9359", + "repositorySequence": 15, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1044, + "balancedV1": 1044, + "qarinahV2": 1044, + "qarinahV2NoGraph": 1044, + "oracle": 1044 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 738, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 738, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 738, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.619915, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "pytest-dev/pytest", + "instanceId": "pytest-dev__pytest-11143", + "repositorySequence": 16, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1044, + "balancedV1": 1044, + "qarinahV2": 1044, + "qarinahV2NoGraph": 1044, + "oracle": 1044 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.394809, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.394809, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.394809, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.394809, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.394809, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 738, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 738, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 738, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.335303, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "pytest-dev/pytest", + "instanceId": "pytest-dev__pytest-11148", + "repositorySequence": 17, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1044, + "balancedV1": 1044, + "qarinahV2": 1044, + "qarinahV2NoGraph": 1044, + "oracle": 1044 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.326663, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "scikit-learn/scikit-learn", + "instanceId": "scikit-learn__scikit-learn-12471", + "repositorySequence": 6, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1778, + "balancedV1": 1778, + "qarinahV2": 1778, + "qarinahV2NoGraph": 1778, + "oracle": 475 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 475, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 893, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 893, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 893, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 893, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 475, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 475, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 475, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 475, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.492532, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 10, + "noTemporalReturnedItems": 10 + }, + { + "repository": "scikit-learn/scikit-learn", + "instanceId": "scikit-learn__scikit-learn-13142", + "repositorySequence": 7, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1778, + "balancedV1": 1778, + "qarinahV2": 1778, + "qarinahV2NoGraph": 1778, + "oracle": 198 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 198, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 893, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 893, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 893, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 893, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 198, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 198, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 198, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 198, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.490734, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 9, + "noTemporalReturnedItems": 10 + }, + { + "repository": "scikit-learn/scikit-learn", + "instanceId": "scikit-learn__scikit-learn-13241", + "repositorySequence": 8, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 1778, + "balancedV1": 1778, + "qarinahV2": 1778, + "qarinahV2NoGraph": 1778, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 418, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 418, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 418, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 418, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 860, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 861, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 860, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 860, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.502273, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "scikit-learn/scikit-learn", + "instanceId": "scikit-learn__scikit-learn-13439", + "repositorySequence": 9, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 1778, + "balancedV1": 1778, + "qarinahV2": 1778, + "qarinahV2NoGraph": 1778, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 418, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 418, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 418, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 418, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 860, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 860, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 860, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 860, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.403103, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 8, + "noTemporalReturnedItems": 10 + }, + { + "repository": "scikit-learn/scikit-learn", + "instanceId": "scikit-learn__scikit-learn-13496", + "repositorySequence": 10, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 1778, + "balancedV1": 1778, + "qarinahV2": 1778, + "qarinahV2NoGraph": 1778, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 442, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 442, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 442, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 442, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 885, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 885, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 885, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 885, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.401596, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "scikit-learn/scikit-learn", + "instanceId": "scikit-learn__scikit-learn-13497", + "repositorySequence": 11, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 1778, + "balancedV1": 1778, + "qarinahV2": 1778, + "qarinahV2NoGraph": 1778, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 442, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 443, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 442, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 442, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 885, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 885, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 885, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 885, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.431, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "SINGLE_OR_NO_SUPPORT", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "scikit-learn/scikit-learn", + "instanceId": "scikit-learn__scikit-learn-13584", + "repositorySequence": 12, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1778, + "balancedV1": 1778, + "qarinahV2": 1778, + "qarinahV2NoGraph": 1778, + "oracle": 418 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 442, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 442, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 442, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 442, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 917, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 917, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 917, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 917, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.540645, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "scikit-learn/scikit-learn", + "instanceId": "scikit-learn__scikit-learn-13779", + "repositorySequence": 13, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.386853, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.386853, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.386853, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.386853, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1778, + "balancedV1": 1778, + "qarinahV2": 1778, + "qarinahV2NoGraph": 1778, + "oracle": 198 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 442, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 442, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 442, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 442, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 198, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 885, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 885, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 885, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 885, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 198, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.386853, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.386853, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.386853, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.386853, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 198, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.386853, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.386853, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.386853, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.386853, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 198, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.386853, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.386853, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.386853, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.386853, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 198, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.528571, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "scikit-learn/scikit-learn", + "instanceId": "scikit-learn__scikit-learn-14087", + "repositorySequence": 14, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.595043, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.595043, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.595043, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.595043, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1778, + "balancedV1": 1778, + "qarinahV2": 1778, + "qarinahV2NoGraph": 1778, + "oracle": 640 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 442, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.275412, + "directRecallAt10": 0, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 442, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.275412, + "directRecallAt10": 0, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 442, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.275412, + "directRecallAt10": 0, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 442, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.275412, + "directRecallAt10": 0, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 198, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.826235, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 860, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.275412, + "directRecallAt10": 0, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 917, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.275412, + "directRecallAt10": 0, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 860, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.275412, + "directRecallAt10": 0, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 860, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.275412, + "directRecallAt10": 0, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 640, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.595043, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.595043, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.595043, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.595043, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 640, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.595043, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.595043, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.595043, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.595043, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 640, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.595043, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.595043, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.595043, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.595043, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 640, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.541929, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "scikit-learn/scikit-learn", + "instanceId": "scikit-learn__scikit-learn-14092", + "repositorySequence": 15, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1778, + "balancedV1": 1778, + "qarinahV2": 1778, + "qarinahV2NoGraph": 1778, + "oracle": 245 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 442, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 442, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 442, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 442, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 245, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 885, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 917, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 885, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 885, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 245, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 245, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 245, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 245, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.34853, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "scikit-learn/scikit-learn", + "instanceId": "scikit-learn__scikit-learn-14894", + "repositorySequence": 16, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 1778, + "balancedV1": 1778, + "qarinahV2": 1778, + "qarinahV2NoGraph": 1778, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 418, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 418, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 418, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 418, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 893, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 893, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 893, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 893, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.515791, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "scikit-learn/scikit-learn", + "instanceId": "scikit-learn__scikit-learn-14983", + "repositorySequence": 17, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 1778, + "balancedV1": 1778, + "qarinahV2": 1778, + "qarinahV2NoGraph": 1778, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 418, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 418, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 418, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 418, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 860, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 860, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 860, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 860, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.492498, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "scikit-learn/scikit-learn", + "instanceId": "scikit-learn__scikit-learn-15512", + "repositorySequence": 18, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 1778, + "balancedV1": 1778, + "qarinahV2": 1778, + "qarinahV2NoGraph": 1778, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 418, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 418, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 418, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 418, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 893, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 893, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 893, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 893, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.434821, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "scikit-learn/scikit-learn", + "instanceId": "scikit-learn__scikit-learn-15535", + "repositorySequence": 19, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 1778, + "balancedV1": 1778, + "qarinahV2": 1778, + "qarinahV2NoGraph": 1778, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 443, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 443, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 443, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 443, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 861, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 861, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 861, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 861, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.563534, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "scikit-learn/scikit-learn", + "instanceId": "scikit-learn__scikit-learn-25500", + "repositorySequence": 20, + "positiveUnderStructuralOracle": true, + "directRecords": 2, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.543771, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.543771, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.543771, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.543771, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1778, + "balancedV1": 1778, + "qarinahV2": 1778, + "qarinahV2NoGraph": 1778, + "oracle": 673 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 475, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 860, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 860, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 860, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 860, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 673, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.543771, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.543771, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.543771, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.543771, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 673, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.543771, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.543771, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.543771, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.543771, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 673, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.543771, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.543771, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.543771, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.543771, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 673, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.4, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "scikit-learn/scikit-learn", + "instanceId": "scikit-learn__scikit-learn-25570", + "repositorySequence": 21, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 1778, + "balancedV1": 1778, + "qarinahV2": 1778, + "qarinahV2NoGraph": 1778, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 475, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 475, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 475, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 475, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 893, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 917, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 893, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 893, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.366775, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "scikit-learn/scikit-learn", + "instanceId": "scikit-learn__scikit-learn-25638", + "repositorySequence": 22, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1778, + "balancedV1": 1778, + "qarinahV2": 1778, + "qarinahV2NoGraph": 1778, + "oracle": 418 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 475, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 893, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 893, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 893, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 893, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.397143, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "scikit-learn/scikit-learn", + "instanceId": "scikit-learn__scikit-learn-25747", + "repositorySequence": 23, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1778, + "balancedV1": 1778, + "qarinahV2": 1778, + "qarinahV2NoGraph": 1778, + "oracle": 418 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 475, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 475, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 475, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 475, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 893, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 917, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 893, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 893, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.333087, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sphinx-doc/sphinx", + "instanceId": "sphinx-doc__sphinx-8273", + "repositorySequence": 4, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 921, + "balancedV1": 921, + "qarinahV2": 921, + "qarinahV2NoGraph": 921, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 418, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 418, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 418, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 418, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.202326, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "NO_CODE_ENTITY_MATCH", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 9, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sphinx-doc/sphinx", + "instanceId": "sphinx-doc__sphinx-8282", + "repositorySequence": 5, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 921, + "balancedV1": 921, + "qarinahV2": 921, + "qarinahV2NoGraph": 921, + "oracle": 743 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 325, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 743, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 743, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 743, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 743, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "DIRECTLY_SUPPORTED", + "decision": "ACCEPT_DIRECT", + "score": 0.67521, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "CONSERVATIVE_DIRECT_THRESHOLD_MET" + ] + }, + "noTemporalFutureItems": 8, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sphinx-doc/sphinx", + "instanceId": "sphinx-doc__sphinx-8435", + "repositorySequence": 6, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 921, + "balancedV1": 921, + "qarinahV2": 921, + "qarinahV2NoGraph": 921, + "oracle": 743 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 325, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 743, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 743, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 743, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 743, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "DIRECTLY_SUPPORTED", + "decision": "ACCEPT_DIRECT", + "score": 0.684694, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "CONSERVATIVE_DIRECT_THRESHOLD_MET" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sphinx-doc/sphinx", + "instanceId": "sphinx-doc__sphinx-8474", + "repositorySequence": 7, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 921, + "balancedV1": 921, + "qarinahV2": 921, + "qarinahV2NoGraph": 921, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 503, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 503, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 503, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 503, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.3625, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sphinx-doc/sphinx", + "instanceId": "sphinx-doc__sphinx-8506", + "repositorySequence": 8, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 921, + "balancedV1": 921, + "qarinahV2": 921, + "qarinahV2NoGraph": 921, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 503, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 503, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 503, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 503, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.35125, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sphinx-doc/sphinx", + "instanceId": "sphinx-doc__sphinx-8595", + "repositorySequence": 9, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 921, + "balancedV1": 921, + "qarinahV2": 921, + "qarinahV2NoGraph": 921, + "oracle": 743 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 325, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 743, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 743, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 743, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 743, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "DIRECTLY_SUPPORTED", + "decision": "ACCEPT_DIRECT", + "score": 0.706591, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "CONSERVATIVE_DIRECT_THRESHOLD_MET" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sphinx-doc/sphinx", + "instanceId": "sphinx-doc__sphinx-8627", + "repositorySequence": 10, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 921, + "balancedV1": 921, + "qarinahV2": 921, + "qarinahV2NoGraph": 921, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 503, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 503, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 503, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 503, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.480249, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sphinx-doc/sphinx", + "instanceId": "sphinx-doc__sphinx-8713", + "repositorySequence": 11, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.96394, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 921, + "balancedV1": 921, + "qarinahV2": 921, + "qarinahV2NoGraph": 921, + "oracle": 743 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 503, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.826235, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 503, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.826235, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 503, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.826235, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 503, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.826235, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 325, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.826235, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.96394, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 743, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.96394, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 743, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.96394, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 743, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.96394, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 743, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.420087, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sphinx-doc/sphinx", + "instanceId": "sphinx-doc__sphinx-8721", + "repositorySequence": 12, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 921, + "balancedV1": 921, + "qarinahV2": 921, + "qarinahV2NoGraph": 921, + "oracle": 743 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 325, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 743, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 743, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 743, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 743, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "DIRECTLY_SUPPORTED", + "decision": "ACCEPT_DIRECT", + "score": 0.694079, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "CONSERVATIVE_DIRECT_THRESHOLD_MET" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sphinx-doc/sphinx", + "instanceId": "sphinx-doc__sphinx-8801", + "repositorySequence": 13, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 921, + "balancedV1": 921, + "qarinahV2": 921, + "qarinahV2NoGraph": 921, + "oracle": 743 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 325, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 743, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 743, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 743, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 743, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.597368, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sphinx-doc/sphinx", + "instanceId": "sphinx-doc__sphinx-10325", + "repositorySequence": 14, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 921, + "balancedV1": 921, + "qarinahV2": 921, + "qarinahV2NoGraph": 921, + "oracle": 743 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 325, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 743, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 743, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 743, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 743, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.283333, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sphinx-doc/sphinx", + "instanceId": "sphinx-doc__sphinx-10451", + "repositorySequence": 15, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 921, + "balancedV1": 921, + "qarinahV2": 921, + "qarinahV2NoGraph": 921, + "oracle": 743 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 503, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 503, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 503, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 503, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 325, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 743, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 743, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 743, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 743, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.325, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "NO_CODE_ENTITY_MATCH", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sphinx-doc/sphinx", + "instanceId": "sphinx-doc__sphinx-11445", + "repositorySequence": 16, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 921, + "balancedV1": 921, + "qarinahV2": 921, + "qarinahV2NoGraph": 921, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 418, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 418, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 418, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 418, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.421595, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-13647", + "repositorySequence": 16, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.618289, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.732829, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.618289, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.618289, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2579, + "balancedV1": 2672, + "qarinahV2": 2579, + "qarinahV2NoGraph": 2579, + "oracle": 948 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 424, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 488, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 424, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 424, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 365, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 904, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.436747, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 975, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.530721, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 904, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.436747, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 904, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.436747, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 948, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1937, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.618289, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1985, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.732829, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1937, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.618289, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1937, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.618289, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 948, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.618289, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.732829, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.618289, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.618289, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 948, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.618289, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.732829, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.618289, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.618289, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 948, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.310811, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "NO_CODE_ENTITY_MATCH", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 8, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-13773", + "repositorySequence": 17, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.598894, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.618891, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.598894, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.598894, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2321, + "balancedV1": 2265, + "qarinahV2": 2321, + "qarinahV2NoGraph": 2321, + "oracle": 948 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 466, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 466, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 466, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 466, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 365, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 973, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 990, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 973, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 973, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 948, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1975, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.463242, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 1919, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.477624, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 1975, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.463242, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1975, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.463242, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 948, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.598894, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.618891, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.598894, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.598894, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 948, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.598894, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.618891, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.598894, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.598894, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 948, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.499857, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 10, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-13895", + "repositorySequence": 18, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.535479, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.546195, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.535479, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.535479, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2579, + "balancedV1": 2374, + "qarinahV2": 2579, + "qarinahV2NoGraph": 2579, + "oracle": 516 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 502, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.152733, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 502, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.152733, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 502, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.152733, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 502, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.152733, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 373, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 952, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.25699, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 952, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.25699, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 952, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.25699, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 952, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.25699, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 516, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1999, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.541869, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1922, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.546195, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1999, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.541869, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1999, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.541869, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 516, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.535479, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.546195, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.535479, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.535479, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 516, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.535479, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.546195, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.535479, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.535479, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 516, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "DIRECTLY_SUPPORTED", + "decision": "ACCEPT_DIRECT", + "score": 0.71875, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "CONSERVATIVE_DIRECT_THRESHOLD_MET" + ] + }, + "noTemporalFutureItems": 8, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-13915", + "repositorySequence": 19, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.391066, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.391066, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.391066, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.391066, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2522, + "balancedV1": 2521, + "qarinahV2": 2522, + "qarinahV2NoGraph": 2522, + "oracle": 516 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 466, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 466, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 466, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 466, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 373, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 986, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.234639, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 986, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.234639, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 986, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.234639, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 986, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.234639, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 516, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.391066, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 1997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.391066, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 1997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.391066, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.391066, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 516, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.391066, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.391066, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.391066, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.391066, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 516, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.391066, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.391066, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.391066, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.391066, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 516, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "DIRECTLY_SUPPORTED", + "decision": "ACCEPT_DIRECT", + "score": 0.678571, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "CONSERVATIVE_DIRECT_THRESHOLD_MET" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-13971", + "repositorySequence": 20, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.515847, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.456762, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.515847, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.515847, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2348, + "balancedV1": 2348, + "qarinahV2": 2348, + "qarinahV2NoGraph": 2348, + "oracle": 578 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 438, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.152733, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 428, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.458199, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 438, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.152733, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 438, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.152733, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 355, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 964, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.515847, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.363114, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 964, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.515847, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 964, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.515847, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.585823, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.526738, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.585823, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.585823, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.515847, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.456762, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.515847, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.515847, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.515847, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.456762, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.515847, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.515847, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.4625, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 8, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-14024", + "repositorySequence": 21, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.546922, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.557405, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.546922, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.546922, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2234, + "balancedV1": 2429, + "qarinahV2": 2234, + "qarinahV2NoGraph": 2234, + "oracle": 516 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 468, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.242076, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 468, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.242076, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 468, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.242076, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 468, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.242076, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 373, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 938, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.335724, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 991, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.335724, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 938, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.335724, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 938, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.335724, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 516, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1983, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.328306, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 1922, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.557405, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1983, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.328306, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1983, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.328306, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 516, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2599, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.546922, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.557405, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.546922, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.546922, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 516, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2599, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.546922, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.557405, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.546922, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.546922, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 516, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.41875, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "NO_CODE_ENTITY_MATCH", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-14308", + "repositorySequence": 22, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.946902, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.871079, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.946902, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.946902, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2556, + "balancedV1": 2556, + "qarinahV2": 2556, + "qarinahV2NoGraph": 2556, + "oracle": 578 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 436, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 428, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 436, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 436, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 365, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 956, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 902, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.703918, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 956, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 956, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1910, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.946902, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1988, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.871079, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1910, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.946902, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1910, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.946902, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.946902, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.871079, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.946902, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.946902, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.946902, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.871079, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.946902, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.946902, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.536071, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-14317", + "repositorySequence": 23, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.97261, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.951834, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.97261, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.97261, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2368, + "balancedV1": 2298, + "qarinahV2": 2368, + "qarinahV2NoGraph": 2368, + "oracle": 578 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 436, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 436, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 436, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 436, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 355, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 891, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.97261, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 964, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 891, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.97261, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 891, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.97261, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1916, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.97261, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1989, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 1916, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.97261, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1916, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.97261, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2535, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.97261, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.951834, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.97261, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.97261, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2535, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.97261, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.951834, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.97261, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.97261, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.475, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "SINGLE_OR_NO_SUPPORT", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-14396", + "repositorySequence": 24, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.356207, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.356207, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.356207, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2398, + "balancedV1": 2604, + "qarinahV2": 2398, + "qarinahV2NoGraph": 2398, + "oracle": 520 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 393, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 474, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 393, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 393, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 0, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 520, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1929, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.356207, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1929, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1929, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.356207, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1929, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.356207, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 520, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.356207, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.356207, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.356207, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 520, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.356207, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.356207, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.356207, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 520, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.343643, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 10, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-14774", + "repositorySequence": 25, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.959654, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.959654, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.959654, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.959654, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2228, + "balancedV1": 2022, + "qarinahV2": 2228, + "qarinahV2NoGraph": 2228, + "oracle": 578 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 436, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 436, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 436, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 436, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 355, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1985, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.959654, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1965, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.959654, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1985, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.959654, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1985, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.959654, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.959654, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.959654, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.959654, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.959654, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.959654, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.959654, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.959654, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.959654, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.394737, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-14817", + "repositorySequence": 26, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.807038, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.477624, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.807038, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.807038, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2142, + "balancedV1": 2252, + "qarinahV2": 2142, + "qarinahV2NoGraph": 2142, + "oracle": 578 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 474, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 508, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 474, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 474, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 365, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 982, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 982, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 982, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 982, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.671386, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.477624, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.671386, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.671386, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.807038, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.477624, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.807038, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.807038, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.807038, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.477624, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.807038, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.807038, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.391422, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 10, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-15011", + "repositorySequence": 27, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 2014, + "balancedV1": 2107, + "qarinahV2": 2014, + "qarinahV2NoGraph": 2014, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 512, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 474, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 512, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 512, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 982, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1000, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 982, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 982, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1844, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1894, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1844, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1844, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.397805, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-15308", + "repositorySequence": 28, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.924133, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1970, + "balancedV1": 1970, + "qarinahV2": 1970, + "qarinahV2NoGraph": 1970, + "oracle": 578 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 436, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 505, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.726229, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 436, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 436, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 355, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 924, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 952, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.830485, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 924, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 924, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.924133, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.924133, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.924133, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.507143, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-15345", + "repositorySequence": 29, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.799101, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.538891, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.799101, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.799101, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2021, + "balancedV1": 2200, + "qarinahV2": 2021, + "qarinahV2NoGraph": 2021, + "oracle": 578 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 488, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.726229, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 488, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.458199, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 488, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.726229, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 488, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.726229, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 365, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 886, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.726229, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 873, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.458199, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 886, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.726229, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 886, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.726229, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1854, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.799101, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 1854, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.538891, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 1854, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.799101, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1854, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.799101, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.799101, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.538891, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.799101, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.799101, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.799101, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.538891, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.799101, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.799101, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.592105, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 9, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-15346", + "repositorySequence": 30, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 2525, + "balancedV1": 2571, + "qarinahV2": 2525, + "qarinahV2NoGraph": 2525, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 468, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 468, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 468, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 468, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 988, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 956, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 988, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 988, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1993, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1993, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1993, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1993, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.45625, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 9, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-15609", + "repositorySequence": 31, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.923633, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.906852, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.923633, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.923633, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2540, + "balancedV1": 2588, + "qarinahV2": 2540, + "qarinahV2NoGraph": 2540, + "oracle": 578 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 464, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.726229, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 464, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.726229, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 464, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.726229, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 464, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.726229, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 355, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 972, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.940915, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 953, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.726229, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 972, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.940915, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 972, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.940915, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1960, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.927959, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1898, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.911177, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1960, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.927959, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1960, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.927959, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.923633, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.906852, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.923633, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.923633, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.923633, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.906852, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.923633, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.923633, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.4, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-15678", + "repositorySequence": 32, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 2150, + "balancedV1": 2251, + "qarinahV2": 2150, + "qarinahV2NoGraph": 2150, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 507, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 507, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 507, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 507, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 953, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 999, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 953, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 953, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1947, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1905, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1947, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1947, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.506522, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 8, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-16106", + "repositorySequence": 33, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.817981, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.625665, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.817981, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.817981, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2353, + "balancedV1": 2604, + "qarinahV2": 2353, + "qarinahV2NoGraph": 2353, + "oracle": 578 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 474, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 474, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 474, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 474, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 365, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.671386, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.498189, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.671386, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.671386, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1979, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.817981, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1918, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.477624, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 1979, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.817981, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1979, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.817981, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.817981, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.625665, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.817981, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.817981, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.817981, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.625665, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.817981, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.817981, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.341176, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 8, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-16281", + "repositorySequence": 34, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.650821, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "balancedV1": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.604931, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.650821, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.650821, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2190, + "balancedV1": 2433, + "qarinahV2": 2190, + "qarinahV2NoGraph": 2190, + "oracle": 578 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 428, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 428, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 428, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 428, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 365, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 934, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 934, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 934, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 934, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1898, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.650821, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 1926, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 1898, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.650821, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1898, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.650821, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3196, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.650821, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.604931, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.650821, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.650821, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3196, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.650821, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.604931, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.650821, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.650821, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.191538, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "NO_CODE_ENTITY_MATCH", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-16503", + "repositorySequence": 35, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.604509, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.444123, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.604509, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.604509, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1938, + "balancedV1": 2047, + "qarinahV2": 1938, + "qarinahV2NoGraph": 1938, + "oracle": 578 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 428, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 428, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 428, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 428, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 365, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 917, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 900, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 917, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 917, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1938, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.604509, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1887, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.444123, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 1938, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.604509, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1938, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.604509, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2676, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.604509, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.444123, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.604509, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.604509, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2676, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.604509, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.444123, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.604509, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.604509, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.446429, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-16792", + "repositorySequence": 36, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 1931, + "balancedV1": 2207, + "qarinahV2": 1931, + "qarinahV2NoGraph": 1931, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 393, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 466, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 393, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 393, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 981, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 981, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 981, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 981, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1931, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1994, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1931, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1931, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.325823, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 8, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-16988", + "repositorySequence": 37, + "positiveUnderStructuralOracle": true, + "directRecords": 2, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.25, + "ndcgAt10": 0.482476, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2396, + "balancedV1": 2319, + "qarinahV2": 2396, + "qarinahV2NoGraph": 2396, + "oracle": 410 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 428, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 428, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 428, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 428, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 410, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.264068, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 410, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 1966, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.25, + "ndcgAt10": 0.482476, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.306574, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 410, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2723, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.25, + "ndcgAt10": 0.482476, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 410, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2723, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.25, + "ndcgAt10": 0.482476, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 410, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.362069, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-17022", + "repositorySequence": 38, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.498189, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.4018, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.498189, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.498189, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2699, + "balancedV1": 2699, + "qarinahV2": 2699, + "qarinahV2NoGraph": 2699, + "oracle": 578 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 505, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 505, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 505, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 505, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 365, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 990, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 990, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.234639, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 990, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 990, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1964, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.498189, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 1942, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.4018, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 1964, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.498189, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1964, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.498189, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.498189, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.4018, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.498189, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.498189, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.498189, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.4018, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.498189, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.498189, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.449051, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-17139", + "repositorySequence": 39, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 2089, + "balancedV1": 2162, + "qarinahV2": 2089, + "qarinahV2NoGraph": 2089, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 373, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 418, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 373, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 373, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 898, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 905, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 898, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 898, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1947, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1992, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1947, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1947, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.421429, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-17630", + "repositorySequence": 40, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.602077, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.592214, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.602077, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.602077, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2609, + "balancedV1": 2637, + "qarinahV2": 2609, + "qarinahV2NoGraph": 2609, + "oracle": 948 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 394, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 466, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 394, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 394, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 346, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.726229, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 962, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.515847, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 962, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.515847, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 962, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.515847, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 962, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.515847, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 948, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.602077, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1920, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.515847, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 1984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.602077, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.602077, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 948, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.602077, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.592214, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.602077, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.602077, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 948, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.602077, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.592214, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.602077, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.602077, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 948, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.404067, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-17655", + "repositorySequence": 41, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 2557, + "balancedV1": 2584, + "qarinahV2": 2557, + "qarinahV2NoGraph": 2557, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 373, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 466, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 373, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 373, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 961, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 962, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 961, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 961, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1970, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1920, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1970, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1970, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.356731, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-18057", + "repositorySequence": 42, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.58816, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.452508, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.58816, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.58816, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2228, + "balancedV1": 2538, + "qarinahV2": 2228, + "qarinahV2NoGraph": 2228, + "oracle": 516 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 474, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 466, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 474, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 474, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 373, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 978, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.477624, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.477624, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 978, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.477624, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 978, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.477624, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 516, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1926, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.452508, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 1991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.593775, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1926, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.452508, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1926, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.452508, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 516, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.58816, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.452508, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.58816, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.58816, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 516, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.58816, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.452508, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.58816, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.58816, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 516, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.372368, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 8, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-18087", + "repositorySequence": 43, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.666373, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.477624, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.666373, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.666373, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2223, + "balancedV1": 2295, + "qarinahV2": 2223, + "qarinahV2NoGraph": 2223, + "oracle": 516 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 411, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 411, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 411, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 411, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 373, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 859, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.530721, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 905, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.477624, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 859, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.530721, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 859, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.530721, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 516, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1877, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.671988, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1918, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.613276, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1877, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.671988, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1877, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.671988, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 516, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2438, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.666373, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.477624, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.666373, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.666373, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 516, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2438, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.666373, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.477624, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.666373, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.666373, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 516, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.286667, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-18189", + "repositorySequence": 44, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 2082, + "balancedV1": 2534, + "qarinahV2": 2534, + "qarinahV2NoGraph": 2534, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 383, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 505, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 505, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 505, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 903, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 903, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 903, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 903, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1717, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1915, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1932, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1932, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2082, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2082, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.585065, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-18199", + "repositorySequence": 45, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 2305, + "balancedV1": 2305, + "qarinahV2": 2305, + "qarinahV2NoGraph": 2305, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 508, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 508, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 508, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 508, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 964, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 901, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 964, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 964, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1984, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1912, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1984, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1984, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2472, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2472, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.468421, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-18532", + "repositorySequence": 46, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.666667, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.308427, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.322809, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.666667, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.308427, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.666667, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.308427, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1970, + "balancedV1": 2098, + "qarinahV2": 1970, + "qarinahV2NoGraph": 1970, + "oracle": 516 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 438, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 438, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 438, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 438, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 373, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 909, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 889, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.181542, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 909, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 909, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 516, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.666667, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.308427, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.322809, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.666667, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.308427, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.666667, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.308427, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 516, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2335, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.666667, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.308427, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.322809, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.666667, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.308427, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.666667, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.308427, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 516, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2335, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.666667, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.308427, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.322809, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.666667, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.308427, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.666667, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.308427, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 516, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.458, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "SINGLE_OR_NO_SUPPORT", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-18621", + "repositorySequence": 47, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.851959, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.860344, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.851959, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.851959, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2601, + "balancedV1": 2601, + "qarinahV2": 2601, + "qarinahV2NoGraph": 2601, + "oracle": 948 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 507, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 508, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 507, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 507, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 365, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 962, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.703918, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 962, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.703918, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 962, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.703918, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 962, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.703918, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 948, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1922, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.851959, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1939, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.860344, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1922, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.851959, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1922, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.851959, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 948, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.851959, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.860344, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.851959, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.851959, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 948, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.851959, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.860344, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.851959, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.851959, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 948, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.524342, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-18698", + "repositorySequence": 48, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2362, + "balancedV1": 2376, + "qarinahV2": 2362, + "qarinahV2NoGraph": 2362, + "oracle": 520 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 507, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 507, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 507, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 507, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 0, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 905, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 905, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 905, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 905, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 520, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1952, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1980, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1952, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1952, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 520, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2908, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 520, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2908, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 520, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.221053, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "NO_CODE_ENTITY_MATCH", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-18835", + "repositorySequence": 49, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 2550, + "balancedV1": 2542, + "qarinahV2": 2550, + "qarinahV2NoGraph": 2550, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 421, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 421, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 421, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 421, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 946, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 946, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 946, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 946, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1955, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1879, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1955, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1955, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.502632, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-19007", + "repositorySequence": 50, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.786472, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.807038, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.786472, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.786472, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2635, + "balancedV1": 2554, + "qarinahV2": 2635, + "qarinahV2NoGraph": 2635, + "oracle": 948 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 489, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 468, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 489, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 489, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 365, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 988, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 896, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 988, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 988, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 948, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1897, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.650821, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 1876, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.671386, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 1897, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.650821, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1897, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.650821, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 948, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.786472, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.807038, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.786472, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.786472, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 948, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.786472, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.807038, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.786472, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.786472, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 948, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "DIRECTLY_SUPPORTED", + "decision": "ACCEPT_DIRECT", + "score": 0.652083, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "CONSERVATIVE_DIRECT_THRESHOLD_MET" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-19254", + "repositorySequence": 51, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.333333, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.315465, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.333333, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.333333, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2570, + "balancedV1": 2584, + "qarinahV2": 2570, + "qarinahV2NoGraph": 2570, + "oracle": 520 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 507, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 507, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 507, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 507, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 0, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 909, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 980, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 909, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 909, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 520, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1987, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.333333, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1973, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1987, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.333333, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1987, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.333333, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 520, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.333333, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.315465, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.333333, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.333333, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 520, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.333333, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.315465, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.333333, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.333333, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 520, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.321631, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-19487", + "repositorySequence": 52, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.573001, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.477624, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.573001, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.573001, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2124, + "balancedV1": 2116, + "qarinahV2": 2124, + "qarinahV2NoGraph": 2124, + "oracle": 542 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 473, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 473, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 473, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 473, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 382, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 985, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.463242, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 985, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.463242, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 985, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.463242, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 542, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1957, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.437349, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 1994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.477624, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 1957, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.437349, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1957, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.437349, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 542, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.573001, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.477624, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.573001, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.573001, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 542, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.573001, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.477624, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.573001, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.573001, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 542, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.214286, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "NO_CODE_ENTITY_MATCH", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-20049", + "repositorySequence": 53, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 2445, + "balancedV1": 2601, + "qarinahV2": 2445, + "qarinahV2NoGraph": 2445, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 464, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 464, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 464, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 464, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 971, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 988, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 971, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 971, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1977, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1996, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1977, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1977, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.330952, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-20154", + "repositorySequence": 54, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 2377, + "balancedV1": 2393, + "qarinahV2": 2377, + "qarinahV2NoGraph": 2377, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 452, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 489, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 452, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 452, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 972, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 891, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 972, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 972, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1983, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1997, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1983, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1983, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.4, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-20212", + "repositorySequence": 55, + "positiveUnderStructuralOracle": true, + "directRecords": 2, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.280062, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.75, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.25, + "ndcgAt10": 0.406372, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.280062, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.280062, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + }, + "oracle": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2298, + "balancedV1": 2407, + "qarinahV2": 2298, + "qarinahV2NoGraph": 2298, + "oracle": 675 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 500, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.108343, + "directRecallAt10": 0, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 507, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.08586, + "directRecallAt10": 0, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2": { + "deliveredTokens": 500, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.108343, + "directRecallAt10": 0, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 500, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.108343, + "directRecallAt10": 0, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 410, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.840185, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 927, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.108343, + "directRecallAt10": 0, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 996, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.25, + "ndcgAt10": 0.140385, + "directRecallAt10": 0, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 927, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.108343, + "directRecallAt10": 0, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 927, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.108343, + "directRecallAt10": 0, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 675, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1900, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.75, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.3297, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.75 + } + }, + "balancedV1": { + "deliveredTokens": 1887, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.75, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.25, + "ndcgAt10": 0.412536, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2": { + "deliveredTokens": 1900, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.75, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.3297, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1900, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.75, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.3297, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.75 + } + }, + "oracle": { + "deliveredTokens": 675, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.280062, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.75, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.25, + "ndcgAt10": 0.406372, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.280062, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.280062, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 675, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.280062, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.75, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.25, + "ndcgAt10": 0.406372, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.280062, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.280062, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 675, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.327143, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "NO_CODE_ENTITY_MATCH", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-20322", + "repositorySequence": 56, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.460123, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.75, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.253691, + "directRecallAt10": 0, + "supportingRecallAt10": 0.75 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.460123, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.460123, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2499, + "balancedV1": 2377, + "qarinahV2": 2499, + "qarinahV2NoGraph": 2499, + "oracle": 881 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 464, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138313, + "directRecallAt10": 0, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 464, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138313, + "directRecallAt10": 0, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2": { + "deliveredTokens": 464, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138313, + "directRecallAt10": 0, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 464, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138313, + "directRecallAt10": 0, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 487, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.795976, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.109611, + "directRecallAt10": 0, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.109611, + "directRecallAt10": 0, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2": { + "deliveredTokens": 984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.109611, + "directRecallAt10": 0, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.109611, + "directRecallAt10": 0, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 881, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.204024, + "directRecallAt10": 0, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 1997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.187699, + "directRecallAt10": 0, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 1991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.204024, + "directRecallAt10": 0, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.204024, + "directRecallAt10": 0, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 881, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.460123, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.75, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.253691, + "directRecallAt10": 0, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.460123, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.460123, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 881, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.460123, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.75, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.253691, + "directRecallAt10": 0, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.460123, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.460123, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 881, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.48419, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-20442", + "repositorySequence": 57, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 2295, + "balancedV1": 2323, + "qarinahV2": 2295, + "qarinahV2NoGraph": 2295, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 508, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 489, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 508, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 508, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 963, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 963, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 963, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 963, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1938, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1995, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1938, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1938, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.196875, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "NO_CODE_ENTITY_MATCH", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-20590", + "repositorySequence": 58, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.61732, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "balancedV1": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + }, + "qarinahV2": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.61732, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.61732, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2606, + "balancedV1": 2707, + "qarinahV2": 2606, + "qarinahV2NoGraph": 2606, + "oracle": 516 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 421, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 421, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 421, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 421, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 373, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 941, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 953, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 941, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 941, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 516, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1925, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 1925, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 1925, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1925, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 516, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.61732, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.61732, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.61732, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 516, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.61732, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.61732, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.61732, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 516, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.406818, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-20639", + "repositorySequence": 59, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.967468, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.946902, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.967468, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.967468, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2383, + "balancedV1": 2588, + "qarinahV2": 2383, + "qarinahV2NoGraph": 2383, + "oracle": 578 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 436, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 436, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 436, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 436, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 365, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 936, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.967468, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 936, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.946902, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 936, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.967468, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 936, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.967468, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1972, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.967468, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1972, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.946902, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1972, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.967468, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1972, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.967468, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3037, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.967468, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.946902, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.967468, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.967468, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3037, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.967468, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.946902, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.967468, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.967468, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.412581, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-21055", + "repositorySequence": 60, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 2208, + "balancedV1": 1901, + "qarinahV2": 2208, + "qarinahV2NoGraph": 2208, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 438, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 507, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 438, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 438, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 887, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 995, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 887, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 887, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1901, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1901, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1901, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1901, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.617021, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-21171", + "repositorySequence": 61, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.515847, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.433677, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.515847, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.515847, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2520, + "balancedV1": 2613, + "qarinahV2": 2520, + "qarinahV2NoGraph": 2520, + "oracle": 578 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 474, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.152733, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 474, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.152733, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 474, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.152733, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 474, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.152733, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 355, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.515847, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.152733, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.515847, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.515847, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1885, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.515847, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 1909, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.433677, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 1885, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.515847, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1885, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.515847, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.515847, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.433677, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.515847, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.515847, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.515847, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.433677, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.515847, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.515847, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.319898, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-21379", + "repositorySequence": 62, + "positiveUnderStructuralOracle": true, + "directRecords": 2, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.828215, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.355222, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + }, + "qarinahV2": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.828215, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.828215, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2244, + "balancedV1": 2424, + "qarinahV2": 2244, + "qarinahV2NoGraph": 2244, + "oracle": 683 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.6235, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 421, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.171719, + "directRecallAt10": 0, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.6235, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.6235, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 432, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.926045, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 971, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.6235, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 927, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.171719, + "directRecallAt10": 0, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2": { + "deliveredTokens": 971, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.6235, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 971, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.6235, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 683, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1979, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.6235, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 1996, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.355222, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 1979, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.6235, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1979, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.6235, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 683, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.828215, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.355222, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.828215, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.828215, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 683, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.828215, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.355222, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.828215, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.828215, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 683, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.349576, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-21612", + "repositorySequence": 63, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.463242, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.416181, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.463242, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.463242, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2437, + "balancedV1": 2474, + "qarinahV2": 2437, + "qarinahV2NoGraph": 2437, + "oracle": 578 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 464, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 464, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 464, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 464, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 365, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.234639, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1985, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.463242, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 1935, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.416181, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 1985, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.463242, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1985, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.463242, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.463242, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.416181, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.463242, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.463242, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.463242, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.416181, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.463242, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.463242, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.4825, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-21614", + "repositorySequence": 64, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.2, + "ndcgAt10": 0.484354, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.2, + "ndcgAt10": 0.47362, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.2, + "ndcgAt10": 0.484354, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.2, + "ndcgAt10": 0.484354, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2589, + "balancedV1": 2589, + "qarinahV2": 2589, + "qarinahV2NoGraph": 2589, + "oracle": 516 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 512, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.234639, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 468, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 512, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.234639, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 512, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.234639, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 373, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 910, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 987, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.202107, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 910, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 910, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 516, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1866, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.348702, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 1987, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.2, + "ndcgAt10": 0.486009, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1866, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.348702, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1866, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.348702, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 516, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.2, + "ndcgAt10": 0.484354, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.2, + "ndcgAt10": 0.47362, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.2, + "ndcgAt10": 0.484354, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.2, + "ndcgAt10": 0.484354, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 516, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.2, + "ndcgAt10": 0.484354, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.2, + "ndcgAt10": 0.47362, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.2, + "ndcgAt10": 0.484354, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.2, + "ndcgAt10": 0.484354, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 516, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.445882, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-21627", + "repositorySequence": 65, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.666667, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.125, + "ndcgAt10": 0.146342, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.666667, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.142848, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.666667, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.125, + "ndcgAt10": 0.146342, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.666667, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.125, + "ndcgAt10": 0.146342, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2504, + "balancedV1": 2504, + "qarinahV2": 2504, + "qarinahV2NoGraph": 2504, + "oracle": 542 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 474, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 474, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 474, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 474, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 382, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 963, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 542, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 542, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2671, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.666667, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.125, + "ndcgAt10": 0.146342, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.666667, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.142848, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.666667, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.125, + "ndcgAt10": 0.146342, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.666667, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.125, + "ndcgAt10": 0.146342, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 542, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2671, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.666667, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.125, + "ndcgAt10": 0.146342, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.666667, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.142848, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.666667, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.125, + "ndcgAt10": 0.146342, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.666667, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.125, + "ndcgAt10": 0.146342, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 542, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.336667, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-21847", + "repositorySequence": 66, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.30103, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.30103, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.30103, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.30103, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2520, + "balancedV1": 2584, + "qarinahV2": 2520, + "qarinahV2NoGraph": 2520, + "oracle": 520 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 393, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 393, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 393, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 393, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 0, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 952, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 952, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 952, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 952, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 520, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1905, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 520, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.30103, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.30103, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.30103, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.30103, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 520, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.30103, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.30103, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.30103, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.30103, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 520, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.334868, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-22005", + "repositorySequence": 67, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 2401, + "balancedV1": 2409, + "qarinahV2": 2401, + "qarinahV2NoGraph": 2401, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 508, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 508, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 508, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 508, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 982, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 982, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 982, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 982, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1887, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1887, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1887, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1887, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.404839, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-22714", + "repositorySequence": 68, + "positiveUnderStructuralOracle": true, + "directRecords": 2, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + }, + "balancedV1": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.806574, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2494, + "balancedV1": 2559, + "qarinahV2": 2494, + "qarinahV2NoGraph": 2494, + "oracle": 410 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 474, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 474, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 474, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 474, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 410, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 982, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.877215, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 982, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 982, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 410, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1974, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 1991, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.81753, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1974, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1974, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 410, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.806574, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 410, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.806574, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 410, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.447143, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-22840", + "repositorySequence": 69, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 2607, + "balancedV1": 2679, + "qarinahV2": 2607, + "qarinahV2NoGraph": 2607, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 512, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 474, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 512, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 512, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 996, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 987, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 996, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 996, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1944, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1944, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1944, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1944, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.388141, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-23117", + "repositorySequence": 70, + "positiveUnderStructuralOracle": true, + "directRecords": 2, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.831555, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.850345, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.831555, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.831555, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2276, + "balancedV1": 2653, + "qarinahV2": 2276, + "qarinahV2NoGraph": 2276, + "oracle": 410 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 474, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 474, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 474, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 410, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.877215, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 942, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.877215, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.877215, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.877215, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 410, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1963, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.831555, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1963, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.850345, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1963, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.831555, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1963, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.831555, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 410, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.831555, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.850345, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.831555, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.831555, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 410, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.831555, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.850345, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.831555, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.831555, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 410, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.405192, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-23191", + "repositorySequence": 71, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.78448, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.786472, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.78448, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.78448, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2369, + "balancedV1": 2369, + "qarinahV2": 2369, + "qarinahV2NoGraph": 2369, + "oracle": 578 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 502, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 464, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 502, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 502, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 365, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 977, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 977, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 977, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 977, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1984, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.78448, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1935, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.650821, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 1984, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.78448, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1984, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.78448, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.78448, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.786472, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.78448, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.78448, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.78448, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.786472, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.78448, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.78448, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.396538, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-23262", + "repositorySequence": 72, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 2606, + "balancedV1": 2605, + "qarinahV2": 2606, + "qarinahV2NoGraph": 2606, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 489, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 489, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 489, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 489, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 925, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 990, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 925, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 925, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1988, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1926, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1988, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1988, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.372308, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-24066", + "repositorySequence": 73, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 2352, + "balancedV1": 2379, + "qarinahV2": 2352, + "qarinahV2NoGraph": 2352, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 373, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 466, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 373, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 373, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 970, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 970, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 970, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 970, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1979, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1996, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1979, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1979, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.346176, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-24102", + "repositorySequence": 74, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 2592, + "balancedV1": 2629, + "qarinahV2": 2592, + "qarinahV2NoGraph": 2592, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 421, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 466, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 421, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 421, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 953, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 998, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 953, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 953, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1911, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1896, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1911, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1911, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.505325, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-24152", + "repositorySequence": 75, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 2534, + "balancedV1": 2552, + "qarinahV2": 2534, + "qarinahV2NoGraph": 2534, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 504, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 504, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 504, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 504, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 943, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 943, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 943, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 943, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1884, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1884, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1884, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1884, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.34875, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-24213", + "repositorySequence": 76, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 2511, + "balancedV1": 2538, + "qarinahV2": 2511, + "qarinahV2NoGraph": 2511, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 474, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 466, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 474, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 474, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 961, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 953, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 961, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 961, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1991, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1991, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1991, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1991, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.444005, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-24909", + "repositorySequence": 77, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 2327, + "balancedV1": 2299, + "qarinahV2": 2327, + "qarinahV2NoGraph": 2327, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 505, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 507, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 505, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 505, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 903, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 944, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 903, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 903, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1997, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1997, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1997, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1997, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.443469, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + } + ], + "onlinePrequential": [ + { + "repository": "astropy/astropy", + "instanceId": "astropy__astropy-7746", + "repositorySequence": 2, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 187, + "balancedV1": 187, + "qarinahV2": 187, + "qarinahV2NoGraph": 187, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 187, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 187, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 187, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 187, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 187, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 187, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 187, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 187, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 187, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 187, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.261375, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 6 + }, + { + "repository": "astropy/astropy", + "instanceId": "astropy__astropy-12907", + "repositorySequence": 3, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 762, + "balancedV1": 762, + "qarinahV2": 762, + "qarinahV2NoGraph": 762, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 187, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 187, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 187, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 762, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 762, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 762, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 762, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 762, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 762, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 762, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 762, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 762, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 762, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 762, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 762, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 762, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 762, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 762, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 762, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.394643, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 6 + }, + { + "repository": "astropy/astropy", + "instanceId": "astropy__astropy-14182", + "repositorySequence": 4, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1140, + "balancedV1": 1140, + "qarinahV2": 1140, + "qarinahV2NoGraph": 1140, + "oracle": 187 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 762, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 762, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 762, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 762, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1140, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1140, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1140, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1140, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1140, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1140, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1140, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1140, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1140, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1140, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1140, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1140, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.37723, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 6 + }, + { + "repository": "astropy/astropy", + "instanceId": "astropy__astropy-14365", + "repositorySequence": 5, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.877215, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.877215, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.877215, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.877215, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1668, + "balancedV1": 1668, + "qarinahV2": 1668, + "qarinahV2NoGraph": 1668, + "oracle": 715 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 378, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 378, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 378, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 378, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 187, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 906, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 906, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 906, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 906, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 715, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1668, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.877215, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1668, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.877215, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1668, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.877215, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1668, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.877215, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 715, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1668, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.877215, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1668, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.877215, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1668, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.877215, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1668, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.877215, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 715, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1668, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.877215, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1668, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.877215, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1668, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.877215, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1668, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.877215, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 715, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.525543, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 6 + }, + { + "repository": "astropy/astropy", + "instanceId": "astropy__astropy-14995", + "repositorySequence": 6, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 2064, + "balancedV1": 2064, + "qarinahV2": 2064, + "qarinahV2NoGraph": 2064, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 396, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 396, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 396, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 396, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 971, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 961, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 971, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 971, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1877, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1877, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1877, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1877, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2064, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 2064, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 2064, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2064, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2064, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 2064, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 2064, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2064, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.439796, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 6 + }, + { + "repository": "django/django", + "instanceId": "django__django-12113", + "repositorySequence": 24, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.456997, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.636364, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 0.5, + "ndcgAt10": 0.631144, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.456997, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.456997, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 4621, + "balancedV1": 3714, + "qarinahV2": 4621, + "qarinahV2NoGraph": 4621, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 881, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 975, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2": { + "deliveredTokens": 881, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 881, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1870, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.31488, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 1980, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.343697, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2": { + "deliveredTokens": 1870, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.31488, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1870, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.31488, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3948, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.454545, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.526428, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "balancedV1": { + "deliveredTokens": 3947, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.636364, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 0.5, + "ndcgAt10": 0.631144, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 3948, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.454545, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.526428, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3948, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.454545, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.526428, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.456997, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.272727, + "recallAt10": 0.636364, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 0.5, + "ndcgAt10": 0.631144, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.456997, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.456997, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.371261, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 9, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-12125", + "repositorySequence": 25, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.5, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.802238, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + }, + "balancedV1": { + "recallAt1": 0.083333, + "recallAt5": 0.25, + "recallAt10": 0.666667, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.644136, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.5, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.802238, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.5, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.802238, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + }, + "oracle": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.833333, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.833333 + } + }, + "volumes": { + "bm25Admitted": 5654, + "balancedV1": 3512, + "qarinahV2": 5654, + "qarinahV2NoGraph": 5654, + "oracle": 3304 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.083333, + "recallAt10": 0.083333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.458466, + "directRecallAt10": 1, + "supportingRecallAt10": 0.083333 + } + }, + "balancedV1": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.083333, + "recallAt10": 0.083333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.458466, + "directRecallAt10": 1, + "supportingRecallAt10": 0.083333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.083333, + "recallAt10": 0.083333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.458466, + "directRecallAt10": 1, + "supportingRecallAt10": 0.083333 + } + }, + "oracle": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.083333, + "recallAt10": 0.083333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.458466, + "directRecallAt10": 1, + "supportingRecallAt10": 0.083333 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 954, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.166667 + } + }, + "balancedV1": { + "deliveredTokens": 997, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.374178, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 954, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 954, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.166667 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.631297, + "directRecallAt10": 1, + "supportingRecallAt10": 0.25 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1964, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.697114, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 1962, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2": { + "deliveredTokens": 1964, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.697114, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1964, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.697114, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.416667, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.756234, + "directRecallAt10": 1, + "supportingRecallAt10": 0.416667 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3874, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.416667, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.756234, + "directRecallAt10": 1, + "supportingRecallAt10": 0.416667 + } + }, + "balancedV1": { + "deliveredTokens": 3938, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.25, + "recallAt10": 0.666667, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.644136, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3874, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.416667, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.756234, + "directRecallAt10": 1, + "supportingRecallAt10": 0.416667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3874, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.416667, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.756234, + "directRecallAt10": 1, + "supportingRecallAt10": 0.416667 + } + }, + "oracle": { + "deliveredTokens": 3442, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.833333, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.833333 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7916, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.5, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.802238, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 7954, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.25, + "recallAt10": 0.666667, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.644136, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 7916, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.5, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.802238, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7916, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.5, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.802238, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 4699, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.833333, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.833333 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.507195, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-12184", + "repositorySequence": 26, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 5281, + "balancedV1": 3605, + "qarinahV2": 5281, + "qarinahV2NoGraph": 5281, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 452, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 459, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 452, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 452, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 943, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 914, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 943, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 943, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1882, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1930, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1882, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1882, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3973, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3900, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3973, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3973, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7948, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 7969, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 7948, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7948, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.543478, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 10, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-12284", + "repositorySequence": 27, + "positiveUnderStructuralOracle": true, + "directRecords": 4, + "supportingRecords": 9, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.076923, + "recallAt5": 0.307692, + "recallAt10": 0.615385, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.647975, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.615385 + }, + "balancedV1": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.615385, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.838569, + "directRecallAt10": 1, + "supportingRecallAt10": 0.615385 + }, + "qarinahV2": { + "recallAt1": 0.076923, + "recallAt5": 0.307692, + "recallAt10": 0.615385, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.647975, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.615385 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.076923, + "recallAt5": 0.307692, + "recallAt10": 0.615385, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.647975, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.615385 + }, + "oracle": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.769231, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.769231 + } + }, + "volumes": { + "bm25Admitted": 3720, + "balancedV1": 3605, + "qarinahV2": 3720, + "qarinahV2NoGraph": 3720, + "oracle": 3493 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.076923, + "recallAt10": 0.076923, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.310341, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.076923 + } + }, + "balancedV1": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.230769, + "recallAt10": 0.230769, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.53078, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.230769 + } + }, + "qarinahV2": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.076923, + "recallAt10": 0.076923, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.310341, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.076923 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.076923, + "recallAt10": 0.076923, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.310341, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.076923 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.153846, + "recallAt10": 0.153846, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.506145, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.153846 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 941, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.153846, + "recallAt10": 0.153846, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.362065, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.153846 + } + }, + "balancedV1": { + "deliveredTokens": 941, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.307692, + "recallAt10": 0.307692, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.56099, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.307692 + } + }, + "qarinahV2": { + "deliveredTokens": 941, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.153846, + "recallAt10": 0.153846, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.362065, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.153846 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 941, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.153846, + "recallAt10": 0.153846, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.362065, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.153846 + } + }, + "oracle": { + "deliveredTokens": 955, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.384615, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.834991, + "directRecallAt10": 1, + "supportingRecallAt10": 0.384615 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1951, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.307692, + "recallAt10": 0.307692, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.446636, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.307692 + } + }, + "balancedV1": { + "deliveredTokens": 1895, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.538462, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.805935, + "directRecallAt10": 1, + "supportingRecallAt10": 0.538462 + } + }, + "qarinahV2": { + "deliveredTokens": 1951, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.307692, + "recallAt10": 0.307692, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.446636, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.307692 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1951, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.307692, + "recallAt10": 0.307692, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.446636, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.307692 + } + }, + "oracle": { + "deliveredTokens": 1933, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.615385, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.938956, + "directRecallAt10": 1, + "supportingRecallAt10": 0.615385 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3967, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.307692, + "recallAt10": 0.615385, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.647975, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.615385 + } + }, + "balancedV1": { + "deliveredTokens": 3900, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.615385, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.838569, + "directRecallAt10": 1, + "supportingRecallAt10": 0.615385 + } + }, + "qarinahV2": { + "deliveredTokens": 3967, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.307692, + "recallAt10": 0.615385, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.647975, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.615385 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3967, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.307692, + "recallAt10": 0.615385, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.647975, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.615385 + } + }, + "oracle": { + "deliveredTokens": 3897, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.769231, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.769231 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7951, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.307692, + "recallAt10": 0.615385, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.647975, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.615385 + } + }, + "balancedV1": { + "deliveredTokens": 7985, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.615385, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.838569, + "directRecallAt10": 1, + "supportingRecallAt10": 0.615385 + } + }, + "qarinahV2": { + "deliveredTokens": 7951, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.307692, + "recallAt10": 0.615385, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.647975, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.615385 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7951, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.307692, + "recallAt10": 0.615385, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.647975, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.615385 + } + }, + "oracle": { + "deliveredTokens": 5154, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.769231, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.769231 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.589706, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-12286", + "repositorySequence": 28, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4526, + "balancedV1": 3642, + "qarinahV2": 4526, + "qarinahV2NoGraph": 4526, + "oracle": 523 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 233, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1872, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 1992, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1872, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1872, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3979, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 3889, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3979, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3979, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7999, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 7998, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7999, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7999, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.237198, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 523, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.453571, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-12308", + "repositorySequence": 29, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.204382, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.204382, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.204382, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4575, + "balancedV1": 3642, + "qarinahV2": 4575, + "qarinahV2NoGraph": 4575, + "oracle": 626 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 251, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 939, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 871, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 939, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 939, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1882, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1953, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1882, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1882, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3977, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.204382, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 3889, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3977, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.204382, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3977, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.204382, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7840, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.204382, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 7981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7840, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.204382, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7840, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.204382, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 626, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.537143, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 10, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-12453", + "repositorySequence": 30, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 14, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.214286, + "recallAt10": 0.214286, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.343697, + "directRecallAt10": null, + "supportingRecallAt10": 0.214286 + }, + "balancedV1": { + "recallAt1": 0.071429, + "recallAt5": 0.285714, + "recallAt10": 0.571429, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.782739, + "directRecallAt10": null, + "supportingRecallAt10": 0.571429 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.214286, + "recallAt10": 0.214286, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.343697, + "directRecallAt10": null, + "supportingRecallAt10": 0.214286 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.214286, + "recallAt10": 0.214286, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.343697, + "directRecallAt10": null, + "supportingRecallAt10": 0.214286 + }, + "oracle": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.714286, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.714286 + } + }, + "volumes": { + "bm25Admitted": 5318, + "balancedV1": 3702, + "qarinahV2": 5318, + "qarinahV2NoGraph": 5318, + "oracle": 3512 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 421, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.071429 + } + }, + "balancedV1": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.071429 + } + }, + "qarinahV2": { + "deliveredTokens": 421, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.071429 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 421, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.071429 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 904, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.071429 + } + }, + "balancedV1": { + "deliveredTokens": 914, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2": { + "deliveredTokens": 904, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.071429 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 904, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.071429 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.214286, + "recallAt10": 0.214286, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.214286 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1911, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.233651, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "balancedV1": { + "deliveredTokens": 1930, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.330138, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2": { + "deliveredTokens": 1911, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.233651, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1911, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.233651, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.357143, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.357143 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3960, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.214286, + "recallAt10": 0.285714, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.409951, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + }, + "balancedV1": { + "deliveredTokens": 3889, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.285714, + "recallAt10": 0.571429, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.782739, + "directRecallAt10": null, + "supportingRecallAt10": 0.571429 + } + }, + "qarinahV2": { + "deliveredTokens": 3960, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.214286, + "recallAt10": 0.285714, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.409951, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3960, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.214286, + "recallAt10": 0.285714, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.409951, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + }, + "oracle": { + "deliveredTokens": 3897, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.714286, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.714286 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7915, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.214286, + "recallAt10": 0.214286, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.343697, + "directRecallAt10": null, + "supportingRecallAt10": 0.214286 + } + }, + "balancedV1": { + "deliveredTokens": 7907, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.285714, + "recallAt10": 0.571429, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.782739, + "directRecallAt10": null, + "supportingRecallAt10": 0.571429 + } + }, + "qarinahV2": { + "deliveredTokens": 7915, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.214286, + "recallAt10": 0.214286, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.343697, + "directRecallAt10": null, + "supportingRecallAt10": 0.214286 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7915, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.214286, + "recallAt10": 0.214286, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.343697, + "directRecallAt10": null, + "supportingRecallAt10": 0.214286 + } + }, + "oracle": { + "deliveredTokens": 5438, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.714286, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.714286 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.374653, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-12470", + "repositorySequence": 31, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 14, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.533333, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.720613, + "directRecallAt10": 1, + "supportingRecallAt10": 0.533333 + }, + "balancedV1": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.533333, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.604176, + "directRecallAt10": 0, + "supportingRecallAt10": 0.533333 + }, + "qarinahV2": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.533333, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.720613, + "directRecallAt10": 1, + "supportingRecallAt10": 0.533333 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.533333, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.720613, + "directRecallAt10": 1, + "supportingRecallAt10": 0.533333 + }, + "oracle": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "volumes": { + "bm25Admitted": 4230, + "balancedV1": 3751, + "qarinahV2": 4230, + "qarinahV2NoGraph": 4230, + "oracle": 4123 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 415, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.133333, + "recallAt10": 0.133333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.133333 + } + }, + "balancedV1": { + "deliveredTokens": 441, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.133333, + "recallAt10": 0.133333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.133333 + } + }, + "qarinahV2": { + "deliveredTokens": 415, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.133333, + "recallAt10": 0.133333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.133333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 415, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.133333, + "recallAt10": 0.133333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.133333 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.133333, + "recallAt10": 0.133333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.133333 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 986, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.325653, + "directRecallAt10": 0, + "supportingRecallAt10": 0.2 + } + }, + "balancedV1": { + "deliveredTokens": 986, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.325653, + "directRecallAt10": 0, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2": { + "deliveredTokens": 986, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.325653, + "directRecallAt10": 0, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 986, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.325653, + "directRecallAt10": 0, + "supportingRecallAt10": 0.2 + } + }, + "oracle": { + "deliveredTokens": 982, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.631297, + "directRecallAt10": 1, + "supportingRecallAt10": 0.2 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1876, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.266667, + "recallAt10": 0.266667, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.523104, + "directRecallAt10": 1, + "supportingRecallAt10": 0.266667 + } + }, + "balancedV1": { + "deliveredTokens": 1995, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.466667, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.555966, + "directRecallAt10": 0, + "supportingRecallAt10": 0.466667 + } + }, + "qarinahV2": { + "deliveredTokens": 1876, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.266667, + "recallAt10": 0.266667, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.523104, + "directRecallAt10": 1, + "supportingRecallAt10": 0.266667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1876, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.266667, + "recallAt10": 0.266667, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.523104, + "directRecallAt10": 1, + "supportingRecallAt10": 0.266667 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.756234, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3926, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.533333, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.720613, + "directRecallAt10": 1, + "supportingRecallAt10": 0.533333 + } + }, + "balancedV1": { + "deliveredTokens": 3889, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.533333, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.604176, + "directRecallAt10": 0, + "supportingRecallAt10": 0.533333 + } + }, + "qarinahV2": { + "deliveredTokens": 3926, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.533333, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.720613, + "directRecallAt10": 1, + "supportingRecallAt10": 0.533333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3926, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.533333, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.720613, + "directRecallAt10": 1, + "supportingRecallAt10": 0.533333 + } + }, + "oracle": { + "deliveredTokens": 3966, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.6, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.955825, + "directRecallAt10": 1, + "supportingRecallAt10": 0.6 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7993, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.533333, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.720613, + "directRecallAt10": 1, + "supportingRecallAt10": 0.533333 + } + }, + "balancedV1": { + "deliveredTokens": 7987, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.533333, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.604176, + "directRecallAt10": 0, + "supportingRecallAt10": 0.533333 + } + }, + "qarinahV2": { + "deliveredTokens": 7993, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.533333, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.720613, + "directRecallAt10": 1, + "supportingRecallAt10": 0.533333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7993, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.533333, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.720613, + "directRecallAt10": 1, + "supportingRecallAt10": 0.533333 + } + }, + "oracle": { + "deliveredTokens": 6236, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.5875, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-12497", + "repositorySequence": 32, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 16, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.0625, + "recallAt5": 0.25, + "recallAt10": 0.4375, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.779171, + "directRecallAt10": null, + "supportingRecallAt10": 0.4375 + }, + "balancedV1": { + "recallAt1": 0.0625, + "recallAt5": 0.3125, + "recallAt10": 0.5, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.860382, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "qarinahV2": { + "recallAt1": 0.0625, + "recallAt5": 0.25, + "recallAt10": 0.4375, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.779171, + "directRecallAt10": null, + "supportingRecallAt10": 0.4375 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.0625, + "recallAt5": 0.25, + "recallAt10": 0.4375, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.779171, + "directRecallAt10": null, + "supportingRecallAt10": 0.4375 + }, + "oracle": { + "recallAt1": 0.0625, + "recallAt5": 0.3125, + "recallAt10": 0.625, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.625 + } + }, + "volumes": { + "bm25Admitted": 4776, + "balancedV1": 4069, + "qarinahV2": 4776, + "qarinahV2NoGraph": 4776, + "oracle": 4123 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 455, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.0625, + "recallAt10": 0.0625, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.0625 + } + }, + "balancedV1": { + "deliveredTokens": 455, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.0625, + "recallAt10": 0.0625, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.0625 + } + }, + "qarinahV2": { + "deliveredTokens": 455, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.0625, + "recallAt10": 0.0625, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.0625 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 455, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.0625, + "recallAt10": 0.0625, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.0625 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 955, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.1875, + "recallAt10": 0.1875, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.1875 + } + }, + "balancedV1": { + "deliveredTokens": 954, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.1875, + "recallAt10": 0.1875, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.1875 + } + }, + "qarinahV2": { + "deliveredTokens": 955, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.1875, + "recallAt10": 0.1875, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.1875 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 955, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.1875, + "recallAt10": 0.1875, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.1875 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.1875, + "recallAt10": 0.1875, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.1875 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1924, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 1937, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.3125, + "recallAt10": 0.375, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.375 + } + }, + "qarinahV2": { + "deliveredTokens": 1924, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1924, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.3125, + "recallAt10": 0.3125, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.3125 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3872, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.25, + "recallAt10": 0.375, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.715551, + "directRecallAt10": null, + "supportingRecallAt10": 0.375 + } + }, + "balancedV1": { + "deliveredTokens": 3972, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.3125, + "recallAt10": 0.5, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.860382, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 3872, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.25, + "recallAt10": 0.375, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.715551, + "directRecallAt10": null, + "supportingRecallAt10": 0.375 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3872, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.25, + "recallAt10": 0.375, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.715551, + "directRecallAt10": null, + "supportingRecallAt10": 0.375 + } + }, + "oracle": { + "deliveredTokens": 3966, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.3125, + "recallAt10": 0.5625, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.936379, + "directRecallAt10": null, + "supportingRecallAt10": 0.5625 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7855, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.25, + "recallAt10": 0.4375, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.779171, + "directRecallAt10": null, + "supportingRecallAt10": 0.4375 + } + }, + "balancedV1": { + "deliveredTokens": 7950, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.3125, + "recallAt10": 0.5, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.860382, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 7855, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.25, + "recallAt10": 0.4375, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.779171, + "directRecallAt10": null, + "supportingRecallAt10": 0.4375 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7855, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.25, + "recallAt10": 0.4375, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.779171, + "directRecallAt10": null, + "supportingRecallAt10": 0.4375 + } + }, + "oracle": { + "deliveredTokens": 6453, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.3125, + "recallAt10": 0.625, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.625 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.497727, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 9, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-12589", + "repositorySequence": 33, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 17, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.058824, + "recallAt5": 0.294118, + "recallAt10": 0.529412, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.926636, + "directRecallAt10": null, + "supportingRecallAt10": 0.529412 + }, + "balancedV1": { + "recallAt1": 0.058824, + "recallAt5": 0.294118, + "recallAt10": 0.470588, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.855348, + "directRecallAt10": null, + "supportingRecallAt10": 0.470588 + }, + "qarinahV2": { + "recallAt1": 0.058824, + "recallAt5": 0.294118, + "recallAt10": 0.529412, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.926636, + "directRecallAt10": null, + "supportingRecallAt10": 0.529412 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.058824, + "recallAt5": 0.294118, + "recallAt10": 0.529412, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.926636, + "directRecallAt10": null, + "supportingRecallAt10": 0.529412 + }, + "oracle": { + "recallAt1": 0.058824, + "recallAt5": 0.294118, + "recallAt10": 0.588235, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.588235 + } + }, + "volumes": { + "bm25Admitted": 4349, + "balancedV1": 4111, + "qarinahV2": 4349, + "qarinahV2NoGraph": 4349, + "oracle": 4220 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 464, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.117647, + "recallAt10": 0.117647, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.117647 + } + }, + "balancedV1": { + "deliveredTokens": 464, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.117647, + "recallAt10": 0.117647, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.117647 + } + }, + "qarinahV2": { + "deliveredTokens": 464, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.117647, + "recallAt10": 0.117647, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.117647 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 464, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.117647, + "recallAt10": 0.117647, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.117647 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.117647, + "recallAt10": 0.117647, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.117647 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 983, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.235294, + "recallAt10": 0.235294, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.235294 + } + }, + "balancedV1": { + "deliveredTokens": 923, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.176471, + "recallAt10": 0.176471, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.176471 + } + }, + "qarinahV2": { + "deliveredTokens": 983, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.235294, + "recallAt10": 0.235294, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.235294 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 983, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.235294, + "recallAt10": 0.235294, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.235294 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.176471, + "recallAt10": 0.176471, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.176471 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1994, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.294118, + "recallAt10": 0.352941, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.352941 + } + }, + "balancedV1": { + "deliveredTokens": 1992, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.294118, + "recallAt10": 0.294118, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.294118 + } + }, + "qarinahV2": { + "deliveredTokens": 1994, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.294118, + "recallAt10": 0.352941, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.352941 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1994, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.294118, + "recallAt10": 0.352941, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.352941 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.294118, + "recallAt10": 0.294118, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.294118 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3884, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.294118, + "recallAt10": 0.529412, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.926636, + "directRecallAt10": null, + "supportingRecallAt10": 0.529412 + } + }, + "balancedV1": { + "deliveredTokens": 3874, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.294118, + "recallAt10": 0.411765, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.791727, + "directRecallAt10": null, + "supportingRecallAt10": 0.411765 + } + }, + "qarinahV2": { + "deliveredTokens": 3884, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.294118, + "recallAt10": 0.529412, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.926636, + "directRecallAt10": null, + "supportingRecallAt10": 0.529412 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3884, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.294118, + "recallAt10": 0.529412, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.926636, + "directRecallAt10": null, + "supportingRecallAt10": 0.529412 + } + }, + "oracle": { + "deliveredTokens": 3916, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.294118, + "recallAt10": 0.588235, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.588235 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7897, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.294118, + "recallAt10": 0.529412, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.926636, + "directRecallAt10": null, + "supportingRecallAt10": 0.529412 + } + }, + "balancedV1": { + "deliveredTokens": 7956, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.294118, + "recallAt10": 0.470588, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.855348, + "directRecallAt10": null, + "supportingRecallAt10": 0.470588 + } + }, + "qarinahV2": { + "deliveredTokens": 7897, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.294118, + "recallAt10": 0.529412, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.926636, + "directRecallAt10": null, + "supportingRecallAt10": 0.529412 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7897, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.294118, + "recallAt10": 0.529412, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.926636, + "directRecallAt10": null, + "supportingRecallAt10": 0.529412 + } + }, + "oracle": { + "deliveredTokens": 6707, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.294118, + "recallAt10": 0.588235, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.588235 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.448293, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-12700", + "repositorySequence": 34, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4979, + "balancedV1": 3990, + "qarinahV2": 4979, + "qarinahV2NoGraph": 4979, + "oracle": 295 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 471, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 471, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 471, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 295, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 964, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 876, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 964, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 964, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 295, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1995, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1977, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1995, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1995, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 295, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 4000, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3990, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 4000, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 4000, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 295, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7937, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 7995, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7937, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7937, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 295, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.419298, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-12708", + "repositorySequence": 35, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 18, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.388889, + "precisionAt5": 0.4, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.710023, + "directRecallAt10": null, + "supportingRecallAt10": 0.388889 + }, + "balancedV1": { + "recallAt1": 0.055556, + "recallAt5": 0.166667, + "recallAt10": 0.333333, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.629364, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + }, + "qarinahV2": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.388889, + "precisionAt5": 0.4, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.710023, + "directRecallAt10": null, + "supportingRecallAt10": 0.388889 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.388889, + "precisionAt5": 0.4, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.710023, + "directRecallAt10": null, + "supportingRecallAt10": 0.388889 + }, + "oracle": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.555556, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.555556 + } + }, + "volumes": { + "bm25Admitted": 4949, + "balancedV1": 4070, + "qarinahV2": 4949, + "qarinahV2NoGraph": 4949, + "oracle": 4236 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.055556, + "recallAt10": 0.055556, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.055556 + } + }, + "balancedV1": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.055556, + "recallAt10": 0.055556, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.055556 + } + }, + "qarinahV2": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.055556, + "recallAt10": 0.055556, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.055556 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.055556, + "recallAt10": 0.055556, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.055556 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.111111 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 929, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.111111 + } + }, + "balancedV1": { + "deliveredTokens": 914, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.111111 + } + }, + "qarinahV2": { + "deliveredTokens": 929, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.111111 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 929, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.111111 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1929, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.166667, + "recallAt10": 0.222222, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.522496, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + }, + "balancedV1": { + "deliveredTokens": 1988, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.055556, + "recallAt10": 0.055556, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.055556 + } + }, + "qarinahV2": { + "deliveredTokens": 1929, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.166667, + "recallAt10": 0.222222, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.522496, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1929, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.166667, + "recallAt10": 0.222222, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.522496, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.277778, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.277778 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3984, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.646402, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 3967, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.166667, + "recallAt10": 0.388889, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.692985, + "directRecallAt10": null, + "supportingRecallAt10": 0.388889 + } + }, + "qarinahV2": { + "deliveredTokens": 3984, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.646402, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3984, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.646402, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 3982, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.5, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.936379, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7999, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.388889, + "precisionAt5": 0.4, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.710023, + "directRecallAt10": null, + "supportingRecallAt10": 0.388889 + } + }, + "balancedV1": { + "deliveredTokens": 7939, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.166667, + "recallAt10": 0.333333, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.629364, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 7999, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.388889, + "precisionAt5": 0.4, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.710023, + "directRecallAt10": null, + "supportingRecallAt10": 0.388889 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7999, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.388889, + "precisionAt5": 0.4, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.710023, + "directRecallAt10": null, + "supportingRecallAt10": 0.388889 + } + }, + "oracle": { + "deliveredTokens": 7371, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.555556, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.555556 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.515385, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 9, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-12747", + "repositorySequence": 36, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 18, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.052632, + "recallAt5": 0.157895, + "recallAt10": 0.368421, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.676237, + "directRecallAt10": 1, + "supportingRecallAt10": 0.368421 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.210526, + "recallAt10": 0.421053, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.497358, + "directRecallAt10": 0, + "supportingRecallAt10": 0.421053 + }, + "qarinahV2": { + "recallAt1": 0.052632, + "recallAt5": 0.157895, + "recallAt10": 0.368421, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.676237, + "directRecallAt10": 1, + "supportingRecallAt10": 0.368421 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.052632, + "recallAt5": 0.157895, + "recallAt10": 0.368421, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.676237, + "directRecallAt10": 1, + "supportingRecallAt10": 0.368421 + }, + "oracle": { + "recallAt1": 0.052632, + "recallAt5": 0.263158, + "recallAt10": 0.526316, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.526316 + } + }, + "volumes": { + "bm25Admitted": 3731, + "balancedV1": 3438, + "qarinahV2": 3731, + "qarinahV2NoGraph": 3731, + "oracle": 4291 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 486, + "metrics": { + "recallAt1": 0.052632, + "recallAt5": 0.157895, + "recallAt10": 0.157895, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.518493, + "directRecallAt10": 1, + "supportingRecallAt10": 0.157895 + } + }, + "balancedV1": { + "deliveredTokens": 512, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.052632, + "recallAt10": 0.052632, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.09642, + "directRecallAt10": 0, + "supportingRecallAt10": 0.052632 + } + }, + "qarinahV2": { + "deliveredTokens": 486, + "metrics": { + "recallAt1": 0.052632, + "recallAt5": 0.157895, + "recallAt10": 0.157895, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.518493, + "directRecallAt10": 1, + "supportingRecallAt10": 0.157895 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 486, + "metrics": { + "recallAt1": 0.052632, + "recallAt5": 0.157895, + "recallAt10": 0.157895, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.518493, + "directRecallAt10": 1, + "supportingRecallAt10": 0.157895 + } + }, + "oracle": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0.052632, + "recallAt5": 0.157895, + "recallAt10": 0.157895, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.631297, + "directRecallAt10": 1, + "supportingRecallAt10": 0.157895 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 929, + "metrics": { + "recallAt1": 0.052632, + "recallAt5": 0.105263, + "recallAt10": 0.105263, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.442082, + "directRecallAt10": 1, + "supportingRecallAt10": 0.105263 + } + }, + "balancedV1": { + "deliveredTokens": 971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.105263, + "recallAt10": 0.105263, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.172831, + "directRecallAt10": 0, + "supportingRecallAt10": 0.105263 + } + }, + "qarinahV2": { + "deliveredTokens": 929, + "metrics": { + "recallAt1": 0.052632, + "recallAt5": 0.105263, + "recallAt10": 0.105263, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.442082, + "directRecallAt10": 1, + "supportingRecallAt10": 0.105263 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 929, + "metrics": { + "recallAt1": 0.052632, + "recallAt5": 0.105263, + "recallAt10": 0.105263, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.442082, + "directRecallAt10": 1, + "supportingRecallAt10": 0.105263 + } + }, + "oracle": { + "deliveredTokens": 984, + "metrics": { + "recallAt1": 0.052632, + "recallAt5": 0.210526, + "recallAt10": 0.210526, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.697114, + "directRecallAt10": 1, + "supportingRecallAt10": 0.210526 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1954, + "metrics": { + "recallAt1": 0.052632, + "recallAt5": 0.157895, + "recallAt10": 0.210526, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.532911, + "directRecallAt10": 1, + "supportingRecallAt10": 0.210526 + } + }, + "balancedV1": { + "deliveredTokens": 1960, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.210526, + "recallAt10": 0.263158, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.352204, + "directRecallAt10": 0, + "supportingRecallAt10": 0.263158 + } + }, + "qarinahV2": { + "deliveredTokens": 1954, + "metrics": { + "recallAt1": 0.052632, + "recallAt5": 0.157895, + "recallAt10": 0.210526, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.532911, + "directRecallAt10": 1, + "supportingRecallAt10": 0.210526 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1954, + "metrics": { + "recallAt1": 0.052632, + "recallAt5": 0.157895, + "recallAt10": 0.210526, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.532911, + "directRecallAt10": 1, + "supportingRecallAt10": 0.210526 + } + }, + "oracle": { + "deliveredTokens": 1917, + "metrics": { + "recallAt1": 0.052632, + "recallAt5": 0.263158, + "recallAt10": 0.368421, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.861611, + "directRecallAt10": 1, + "supportingRecallAt10": 0.368421 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3968, + "metrics": { + "recallAt1": 0.052632, + "recallAt5": 0.157895, + "recallAt10": 0.368421, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.676237, + "directRecallAt10": 1, + "supportingRecallAt10": 0.368421 + } + }, + "balancedV1": { + "deliveredTokens": 3997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.210526, + "recallAt10": 0.421053, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.497358, + "directRecallAt10": 0, + "supportingRecallAt10": 0.421053 + } + }, + "qarinahV2": { + "deliveredTokens": 3968, + "metrics": { + "recallAt1": 0.052632, + "recallAt5": 0.157895, + "recallAt10": 0.368421, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.676237, + "directRecallAt10": 1, + "supportingRecallAt10": 0.368421 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3968, + "metrics": { + "recallAt1": 0.052632, + "recallAt5": 0.157895, + "recallAt10": 0.368421, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.676237, + "directRecallAt10": 1, + "supportingRecallAt10": 0.368421 + } + }, + "oracle": { + "deliveredTokens": 3881, + "metrics": { + "recallAt1": 0.052632, + "recallAt5": 0.263158, + "recallAt10": 0.526316, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.526316 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7876, + "metrics": { + "recallAt1": 0.052632, + "recallAt5": 0.157895, + "recallAt10": 0.368421, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.676237, + "directRecallAt10": 1, + "supportingRecallAt10": 0.368421 + } + }, + "balancedV1": { + "deliveredTokens": 7759, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.210526, + "recallAt10": 0.421053, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.497358, + "directRecallAt10": 0, + "supportingRecallAt10": 0.421053 + } + }, + "qarinahV2": { + "deliveredTokens": 7876, + "metrics": { + "recallAt1": 0.052632, + "recallAt5": 0.157895, + "recallAt10": 0.368421, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.676237, + "directRecallAt10": 1, + "supportingRecallAt10": 0.368421 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7876, + "metrics": { + "recallAt1": 0.052632, + "recallAt5": 0.157895, + "recallAt10": 0.368421, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.676237, + "directRecallAt10": 1, + "supportingRecallAt10": 0.368421 + } + }, + "oracle": { + "deliveredTokens": 7680, + "metrics": { + "recallAt1": 0.052632, + "recallAt5": 0.263158, + "recallAt10": 0.526316, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.526316 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.358, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "NO_CODE_ENTITY_MATCH", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 8, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-12856", + "repositorySequence": 37, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 20, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.05, + "recallAt5": 0.2, + "recallAt10": 0.4, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.845426, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + }, + "balancedV1": { + "recallAt1": 0.05, + "recallAt5": 0.25, + "recallAt10": 0.4, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.863015, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + }, + "qarinahV2": { + "recallAt1": 0.05, + "recallAt5": 0.2, + "recallAt10": 0.4, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.845426, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.05, + "recallAt5": 0.2, + "recallAt10": 0.4, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.845426, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + }, + "oracle": { + "recallAt1": 0.05, + "recallAt5": 0.25, + "recallAt10": 0.5, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "volumes": { + "bm25Admitted": 3116, + "balancedV1": 3190, + "qarinahV2": 3116, + "qarinahV2NoGraph": 3116, + "oracle": 4291 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 447, + "metrics": { + "recallAt1": 0.05, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.1 + } + }, + "balancedV1": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0.05, + "recallAt5": 0.15, + "recallAt10": 0.15, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.15 + } + }, + "qarinahV2": { + "deliveredTokens": 447, + "metrics": { + "recallAt1": 0.05, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 447, + "metrics": { + "recallAt1": 0.05, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.1 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.05, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 918, + "metrics": { + "recallAt1": 0.05, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "balancedV1": { + "deliveredTokens": 985, + "metrics": { + "recallAt1": 0.05, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2": { + "deliveredTokens": 918, + "metrics": { + "recallAt1": 0.05, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 918, + "metrics": { + "recallAt1": 0.05, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "oracle": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0.05, + "recallAt5": 0.15, + "recallAt10": 0.15, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.15 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1969, + "metrics": { + "recallAt1": 0.05, + "recallAt5": 0.2, + "recallAt10": 0.25, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.642187, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 1997, + "metrics": { + "recallAt1": 0.05, + "recallAt5": 0.25, + "recallAt10": 0.3, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.3 + } + }, + "qarinahV2": { + "deliveredTokens": 1969, + "metrics": { + "recallAt1": 0.05, + "recallAt5": 0.2, + "recallAt10": 0.25, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.642187, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1969, + "metrics": { + "recallAt1": 0.05, + "recallAt5": 0.2, + "recallAt10": 0.25, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.642187, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.05, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3884, + "metrics": { + "recallAt1": 0.05, + "recallAt5": 0.2, + "recallAt10": 0.4, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.845426, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + }, + "balancedV1": { + "deliveredTokens": 3985, + "metrics": { + "recallAt1": 0.05, + "recallAt5": 0.25, + "recallAt10": 0.4, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.863015, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2": { + "deliveredTokens": 3884, + "metrics": { + "recallAt1": 0.05, + "recallAt5": 0.2, + "recallAt10": 0.4, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.845426, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3884, + "metrics": { + "recallAt1": 0.05, + "recallAt5": 0.2, + "recallAt10": 0.4, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.845426, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + }, + "oracle": { + "deliveredTokens": 3881, + "metrics": { + "recallAt1": 0.05, + "recallAt5": 0.25, + "recallAt10": 0.5, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7904, + "metrics": { + "recallAt1": 0.05, + "recallAt5": 0.2, + "recallAt10": 0.4, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.845426, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + }, + "balancedV1": { + "deliveredTokens": 7916, + "metrics": { + "recallAt1": 0.05, + "recallAt5": 0.25, + "recallAt10": 0.4, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.863015, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2": { + "deliveredTokens": 7904, + "metrics": { + "recallAt1": 0.05, + "recallAt5": 0.2, + "recallAt10": 0.4, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.845426, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7904, + "metrics": { + "recallAt1": 0.05, + "recallAt5": 0.2, + "recallAt10": 0.4, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.845426, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + }, + "oracle": { + "deliveredTokens": 7969, + "metrics": { + "recallAt1": 0.05, + "recallAt5": 0.25, + "recallAt10": 0.5, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.535, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-12908", + "repositorySequence": 38, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 21, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.047619, + "recallAt5": 0.190476, + "recallAt10": 0.380952, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.826333, + "directRecallAt10": null, + "supportingRecallAt10": 0.380952 + }, + "balancedV1": { + "recallAt1": 0.047619, + "recallAt5": 0.238095, + "recallAt10": 0.47619, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.47619 + }, + "qarinahV2": { + "recallAt1": 0.047619, + "recallAt5": 0.190476, + "recallAt10": 0.380952, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.826333, + "directRecallAt10": null, + "supportingRecallAt10": 0.380952 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.047619, + "recallAt5": 0.190476, + "recallAt10": 0.380952, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.826333, + "directRecallAt10": null, + "supportingRecallAt10": 0.380952 + }, + "oracle": { + "recallAt1": 0.047619, + "recallAt5": 0.238095, + "recallAt10": 0.47619, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.47619 + } + }, + "volumes": { + "bm25Admitted": 3901, + "balancedV1": 3018, + "qarinahV2": 3901, + "qarinahV2NoGraph": 3901, + "oracle": 3798 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.047619, + "recallAt5": 0.047619, + "recallAt10": 0.047619, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.047619 + } + }, + "balancedV1": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.047619, + "recallAt5": 0.047619, + "recallAt10": 0.047619, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.047619 + } + }, + "qarinahV2": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.047619, + "recallAt5": 0.047619, + "recallAt10": 0.047619, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.047619 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.047619, + "recallAt5": 0.047619, + "recallAt10": 0.047619, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.047619 + } + }, + "oracle": { + "deliveredTokens": 408, + "metrics": { + "recallAt1": 0.047619, + "recallAt5": 0.095238, + "recallAt10": 0.095238, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.095238 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 881, + "metrics": { + "recallAt1": 0.047619, + "recallAt5": 0.047619, + "recallAt10": 0.047619, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.047619 + } + }, + "balancedV1": { + "deliveredTokens": 994, + "metrics": { + "recallAt1": 0.047619, + "recallAt5": 0.190476, + "recallAt10": 0.190476, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.190476 + } + }, + "qarinahV2": { + "deliveredTokens": 881, + "metrics": { + "recallAt1": 0.047619, + "recallAt5": 0.047619, + "recallAt10": 0.047619, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.047619 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 881, + "metrics": { + "recallAt1": 0.047619, + "recallAt5": 0.047619, + "recallAt10": 0.047619, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.047619 + } + }, + "oracle": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.047619, + "recallAt5": 0.190476, + "recallAt10": 0.190476, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.190476 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1954, + "metrics": { + "recallAt1": 0.047619, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.453743, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "balancedV1": { + "deliveredTokens": 1983, + "metrics": { + "recallAt1": 0.047619, + "recallAt5": 0.238095, + "recallAt10": 0.285714, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + }, + "qarinahV2": { + "deliveredTokens": 1954, + "metrics": { + "recallAt1": 0.047619, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.453743, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1954, + "metrics": { + "recallAt1": 0.047619, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.453743, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "oracle": { + "deliveredTokens": 1931, + "metrics": { + "recallAt1": 0.047619, + "recallAt5": 0.238095, + "recallAt10": 0.333333, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.800694, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3901, + "metrics": { + "recallAt1": 0.047619, + "recallAt5": 0.190476, + "recallAt10": 0.380952, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.826333, + "directRecallAt10": null, + "supportingRecallAt10": 0.380952 + } + }, + "balancedV1": { + "deliveredTokens": 3906, + "metrics": { + "recallAt1": 0.047619, + "recallAt5": 0.238095, + "recallAt10": 0.47619, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.47619 + } + }, + "qarinahV2": { + "deliveredTokens": 3901, + "metrics": { + "recallAt1": 0.047619, + "recallAt5": 0.190476, + "recallAt10": 0.380952, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.826333, + "directRecallAt10": null, + "supportingRecallAt10": 0.380952 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3901, + "metrics": { + "recallAt1": 0.047619, + "recallAt5": 0.190476, + "recallAt10": 0.380952, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.826333, + "directRecallAt10": null, + "supportingRecallAt10": 0.380952 + } + }, + "oracle": { + "deliveredTokens": 3955, + "metrics": { + "recallAt1": 0.047619, + "recallAt5": 0.238095, + "recallAt10": 0.47619, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.47619 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7864, + "metrics": { + "recallAt1": 0.047619, + "recallAt5": 0.190476, + "recallAt10": 0.380952, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.826333, + "directRecallAt10": null, + "supportingRecallAt10": 0.380952 + } + }, + "balancedV1": { + "deliveredTokens": 7948, + "metrics": { + "recallAt1": 0.047619, + "recallAt5": 0.238095, + "recallAt10": 0.47619, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.47619 + } + }, + "qarinahV2": { + "deliveredTokens": 7864, + "metrics": { + "recallAt1": 0.047619, + "recallAt5": 0.190476, + "recallAt10": 0.380952, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.826333, + "directRecallAt10": null, + "supportingRecallAt10": 0.380952 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7864, + "metrics": { + "recallAt1": 0.047619, + "recallAt5": 0.190476, + "recallAt10": 0.380952, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.826333, + "directRecallAt10": null, + "supportingRecallAt10": 0.380952 + } + }, + "oracle": { + "deliveredTokens": 7856, + "metrics": { + "recallAt1": 0.047619, + "recallAt5": 0.238095, + "recallAt10": 0.47619, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.47619 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.4425, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 8, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-12915", + "repositorySequence": 39, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.156426, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.156426, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.156426, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 5408, + "balancedV1": 3973, + "qarinahV2": 5408, + "qarinahV2NoGraph": 5408, + "oracle": 802 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 472, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 472, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 472, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 427, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 977, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.234639, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 875, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 977, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.234639, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 977, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.234639, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 802, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1967, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1880, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1967, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1967, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 802, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3917, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.16716, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 3973, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3917, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.16716, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3917, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.16716, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 802, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7973, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.156426, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 7990, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7973, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.156426, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7973, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.156426, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 802, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.566384, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-12983", + "repositorySequence": 40, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.148041, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.375906, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.148041, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.148041, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4875, + "balancedV1": 3938, + "qarinahV2": 4875, + "qarinahV2NoGraph": 4875, + "oracle": 1678 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 469, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 469, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 469, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 444, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 902, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 983, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.234639, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 902, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 902, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 444, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1982, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1901, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.234639, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 1982, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1982, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 1678, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3926, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.156426, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 3938, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.375906, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3926, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.156426, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3926, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.156426, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 1678, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7911, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.148041, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 7909, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.375906, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 7911, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.148041, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7911, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.148041, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 1678, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.528571, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13028", + "repositorySequence": 41, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 21, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.045455, + "recallAt5": 0.227273, + "recallAt10": 0.363636, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.720613, + "directRecallAt10": 1, + "supportingRecallAt10": 0.363636 + }, + "balancedV1": { + "recallAt1": 0.045455, + "recallAt5": 0.227273, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.694356, + "directRecallAt10": 0, + "supportingRecallAt10": 0.454545 + }, + "qarinahV2": { + "recallAt1": 0.045455, + "recallAt5": 0.227273, + "recallAt10": 0.363636, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.720613, + "directRecallAt10": 1, + "supportingRecallAt10": 0.363636 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.045455, + "recallAt5": 0.227273, + "recallAt10": 0.363636, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.720613, + "directRecallAt10": 1, + "supportingRecallAt10": 0.363636 + }, + "oracle": { + "recallAt1": 0.045455, + "recallAt5": 0.227273, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + }, + "volumes": { + "bm25Admitted": 6303, + "balancedV1": 3119, + "qarinahV2": 6303, + "qarinahV2NoGraph": 6303, + "oracle": 4190 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 500, + "metrics": { + "recallAt1": 0.045455, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.045455, + "recallAt5": 0.045455, + "recallAt10": 0.045455, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.045455 + } + }, + "qarinahV2": { + "deliveredTokens": 500, + "metrics": { + "recallAt1": 0.045455, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 500, + "metrics": { + "recallAt1": 0.045455, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 408, + "metrics": { + "recallAt1": 0.045455, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 978, + "metrics": { + "recallAt1": 0.045455, + "recallAt5": 0.136364, + "recallAt10": 0.136364, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.325653, + "directRecallAt10": 0, + "supportingRecallAt10": 0.136364 + } + }, + "balancedV1": { + "deliveredTokens": 914, + "metrics": { + "recallAt1": 0.045455, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2": { + "deliveredTokens": 978, + "metrics": { + "recallAt1": 0.045455, + "recallAt5": 0.136364, + "recallAt10": 0.136364, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.325653, + "directRecallAt10": 0, + "supportingRecallAt10": 0.136364 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 978, + "metrics": { + "recallAt1": 0.045455, + "recallAt5": 0.136364, + "recallAt10": 0.136364, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.325653, + "directRecallAt10": 0, + "supportingRecallAt10": 0.136364 + } + }, + "oracle": { + "deliveredTokens": 901, + "metrics": { + "recallAt1": 0.045455, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.090909 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1915, + "metrics": { + "recallAt1": 0.045455, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 1902, + "metrics": { + "recallAt1": 0.045455, + "recallAt5": 0.227273, + "recallAt10": 0.227273, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.450589, + "directRecallAt10": 0, + "supportingRecallAt10": 0.227273 + } + }, + "qarinahV2": { + "deliveredTokens": 1915, + "metrics": { + "recallAt1": 0.045455, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1915, + "metrics": { + "recallAt1": 0.045455, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 1893, + "metrics": { + "recallAt1": 0.045455, + "recallAt5": 0.227273, + "recallAt10": 0.227273, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.756234, + "directRecallAt10": 1, + "supportingRecallAt10": 0.227273 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 4000, + "metrics": { + "recallAt1": 0.045455, + "recallAt5": 0.227273, + "recallAt10": 0.272727, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.633164, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 3996, + "metrics": { + "recallAt1": 0.045455, + "recallAt5": 0.227273, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.694356, + "directRecallAt10": 0, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2": { + "deliveredTokens": 4000, + "metrics": { + "recallAt1": 0.045455, + "recallAt5": 0.227273, + "recallAt10": 0.272727, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.633164, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 4000, + "metrics": { + "recallAt1": 0.045455, + "recallAt5": 0.227273, + "recallAt10": 0.272727, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.633164, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 3876, + "metrics": { + "recallAt1": 0.045455, + "recallAt5": 0.227273, + "recallAt10": 0.409091, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.955825, + "directRecallAt10": 1, + "supportingRecallAt10": 0.409091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7999, + "metrics": { + "recallAt1": 0.045455, + "recallAt5": 0.227273, + "recallAt10": 0.363636, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.720613, + "directRecallAt10": 1, + "supportingRecallAt10": 0.363636 + } + }, + "balancedV1": { + "deliveredTokens": 7891, + "metrics": { + "recallAt1": 0.045455, + "recallAt5": 0.227273, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.694356, + "directRecallAt10": 0, + "supportingRecallAt10": 0.454545 + } + }, + "qarinahV2": { + "deliveredTokens": 7999, + "metrics": { + "recallAt1": 0.045455, + "recallAt5": 0.227273, + "recallAt10": 0.363636, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.720613, + "directRecallAt10": 1, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7999, + "metrics": { + "recallAt1": 0.045455, + "recallAt5": 0.227273, + "recallAt10": 0.363636, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.720613, + "directRecallAt10": 1, + "supportingRecallAt10": 0.363636 + } + }, + "oracle": { + "deliveredTokens": 7881, + "metrics": { + "recallAt1": 0.045455, + "recallAt5": 0.227273, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.443568, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 9, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13033", + "repositorySequence": 42, + "positiveUnderStructuralOracle": true, + "directRecords": 2, + "supportingRecords": 21, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.043478, + "recallAt5": 0.217391, + "recallAt10": 0.434783, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.434783 + }, + "balancedV1": { + "recallAt1": 0.043478, + "recallAt5": 0.217391, + "recallAt10": 0.391304, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.962966, + "directRecallAt10": 1, + "supportingRecallAt10": 0.391304 + }, + "qarinahV2": { + "recallAt1": 0.043478, + "recallAt5": 0.217391, + "recallAt10": 0.434783, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.434783 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.043478, + "recallAt5": 0.217391, + "recallAt10": 0.434783, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.434783 + }, + "oracle": { + "recallAt1": 0.043478, + "recallAt5": 0.217391, + "recallAt10": 0.434783, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.434783 + } + }, + "volumes": { + "bm25Admitted": 4771, + "balancedV1": 5227, + "qarinahV2": 4771, + "qarinahV2NoGraph": 4771, + "oracle": 4203 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 501, + "metrics": { + "recallAt1": 0.043478, + "recallAt5": 0.086957, + "recallAt10": 0.086957, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.465181, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.086957 + } + }, + "balancedV1": { + "deliveredTokens": 454, + "metrics": { + "recallAt1": 0.043478, + "recallAt5": 0.086957, + "recallAt10": 0.086957, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.465181, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.086957 + } + }, + "qarinahV2": { + "deliveredTokens": 501, + "metrics": { + "recallAt1": 0.043478, + "recallAt5": 0.086957, + "recallAt10": 0.086957, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.465181, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.086957 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 501, + "metrics": { + "recallAt1": 0.043478, + "recallAt5": 0.086957, + "recallAt10": 0.086957, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.465181, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.086957 + } + }, + "oracle": { + "deliveredTokens": 454, + "metrics": { + "recallAt1": 0.043478, + "recallAt5": 0.086957, + "recallAt10": 0.086957, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.465181, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.086957 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 988, + "metrics": { + "recallAt1": 0.043478, + "recallAt5": 0.130435, + "recallAt10": 0.130435, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.690903, + "directRecallAt10": 1, + "supportingRecallAt10": 0.130435 + } + }, + "balancedV1": { + "deliveredTokens": 988, + "metrics": { + "recallAt1": 0.043478, + "recallAt5": 0.130435, + "recallAt10": 0.130435, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.690903, + "directRecallAt10": 1, + "supportingRecallAt10": 0.130435 + } + }, + "qarinahV2": { + "deliveredTokens": 988, + "metrics": { + "recallAt1": 0.043478, + "recallAt5": 0.130435, + "recallAt10": 0.130435, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.690903, + "directRecallAt10": 1, + "supportingRecallAt10": 0.130435 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 988, + "metrics": { + "recallAt1": 0.043478, + "recallAt5": 0.130435, + "recallAt10": 0.130435, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.690903, + "directRecallAt10": 1, + "supportingRecallAt10": 0.130435 + } + }, + "oracle": { + "deliveredTokens": 962, + "metrics": { + "recallAt1": 0.043478, + "recallAt5": 0.130435, + "recallAt10": 0.130435, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.690903, + "directRecallAt10": 1, + "supportingRecallAt10": 0.130435 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1988, + "metrics": { + "recallAt1": 0.043478, + "recallAt5": 0.217391, + "recallAt10": 0.217391, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.795642, + "directRecallAt10": 1, + "supportingRecallAt10": 0.217391 + } + }, + "balancedV1": { + "deliveredTokens": 1950, + "metrics": { + "recallAt1": 0.043478, + "recallAt5": 0.173913, + "recallAt10": 0.173913, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.74608, + "directRecallAt10": 1, + "supportingRecallAt10": 0.173913 + } + }, + "qarinahV2": { + "deliveredTokens": 1988, + "metrics": { + "recallAt1": 0.043478, + "recallAt5": 0.217391, + "recallAt10": 0.217391, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.795642, + "directRecallAt10": 1, + "supportingRecallAt10": 0.217391 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1988, + "metrics": { + "recallAt1": 0.043478, + "recallAt5": 0.217391, + "recallAt10": 0.217391, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.795642, + "directRecallAt10": 1, + "supportingRecallAt10": 0.217391 + } + }, + "oracle": { + "deliveredTokens": 1960, + "metrics": { + "recallAt1": 0.043478, + "recallAt5": 0.217391, + "recallAt10": 0.26087, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.841278, + "directRecallAt10": 1, + "supportingRecallAt10": 0.26087 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3973, + "metrics": { + "recallAt1": 0.043478, + "recallAt5": 0.217391, + "recallAt10": 0.391304, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.962966, + "directRecallAt10": 1, + "supportingRecallAt10": 0.391304 + } + }, + "balancedV1": { + "deliveredTokens": 3993, + "metrics": { + "recallAt1": 0.043478, + "recallAt5": 0.217391, + "recallAt10": 0.391304, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.962966, + "directRecallAt10": 1, + "supportingRecallAt10": 0.391304 + } + }, + "qarinahV2": { + "deliveredTokens": 3973, + "metrics": { + "recallAt1": 0.043478, + "recallAt5": 0.217391, + "recallAt10": 0.391304, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.962966, + "directRecallAt10": 1, + "supportingRecallAt10": 0.391304 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3973, + "metrics": { + "recallAt1": 0.043478, + "recallAt5": 0.217391, + "recallAt10": 0.391304, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.962966, + "directRecallAt10": 1, + "supportingRecallAt10": 0.391304 + } + }, + "oracle": { + "deliveredTokens": 3874, + "metrics": { + "recallAt1": 0.043478, + "recallAt5": 0.217391, + "recallAt10": 0.434783, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.434783 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7973, + "metrics": { + "recallAt1": 0.043478, + "recallAt5": 0.217391, + "recallAt10": 0.434783, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.434783 + } + }, + "balancedV1": { + "deliveredTokens": 7853, + "metrics": { + "recallAt1": 0.043478, + "recallAt5": 0.217391, + "recallAt10": 0.391304, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.962966, + "directRecallAt10": 1, + "supportingRecallAt10": 0.391304 + } + }, + "qarinahV2": { + "deliveredTokens": 7973, + "metrics": { + "recallAt1": 0.043478, + "recallAt5": 0.217391, + "recallAt10": 0.434783, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.434783 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7973, + "metrics": { + "recallAt1": 0.043478, + "recallAt5": 0.217391, + "recallAt10": 0.434783, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.434783 + } + }, + "oracle": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.043478, + "recallAt5": 0.217391, + "recallAt10": 0.434783, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.434783 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.429664, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13158", + "repositorySequence": 43, + "positiveUnderStructuralOracle": true, + "directRecords": 2, + "supportingRecords": 22, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.041667, + "recallAt5": 0.208333, + "recallAt10": 0.375, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.783867, + "directRecallAt10": 1, + "supportingRecallAt10": 0.375 + }, + "balancedV1": { + "recallAt1": 0.041667, + "recallAt5": 0.208333, + "recallAt10": 0.416667, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.777867, + "directRecallAt10": 1, + "supportingRecallAt10": 0.416667 + }, + "qarinahV2": { + "recallAt1": 0.041667, + "recallAt5": 0.208333, + "recallAt10": 0.375, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.783867, + "directRecallAt10": 1, + "supportingRecallAt10": 0.375 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.041667, + "recallAt5": 0.208333, + "recallAt10": 0.375, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.783867, + "directRecallAt10": 1, + "supportingRecallAt10": 0.375 + }, + "oracle": { + "recallAt1": 0.041667, + "recallAt5": 0.208333, + "recallAt10": 0.416667, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.416667 + } + }, + "volumes": { + "bm25Admitted": 4916, + "balancedV1": 4941, + "qarinahV2": 4916, + "qarinahV2NoGraph": 4916, + "oracle": 5339 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 501, + "metrics": { + "recallAt1": 0.041667, + "recallAt5": 0.083333, + "recallAt10": 0.083333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.208948, + "directRecallAt10": 0, + "supportingRecallAt10": 0.083333 + } + }, + "balancedV1": { + "deliveredTokens": 501, + "metrics": { + "recallAt1": 0.041667, + "recallAt5": 0.083333, + "recallAt10": 0.083333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.208948, + "directRecallAt10": 0, + "supportingRecallAt10": 0.083333 + } + }, + "qarinahV2": { + "deliveredTokens": 501, + "metrics": { + "recallAt1": 0.041667, + "recallAt5": 0.083333, + "recallAt10": 0.083333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.208948, + "directRecallAt10": 0, + "supportingRecallAt10": 0.083333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 501, + "metrics": { + "recallAt1": 0.041667, + "recallAt5": 0.083333, + "recallAt10": 0.083333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.208948, + "directRecallAt10": 0, + "supportingRecallAt10": 0.083333 + } + }, + "oracle": { + "deliveredTokens": 408, + "metrics": { + "recallAt1": 0.041667, + "recallAt5": 0.083333, + "recallAt10": 0.083333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.208948, + "directRecallAt10": 0, + "supportingRecallAt10": 0.083333 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.041667, + "recallAt5": 0.041667, + "recallAt10": 0.041667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.128116, + "directRecallAt10": 0, + "supportingRecallAt10": 0.041667 + } + }, + "balancedV1": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.041667, + "recallAt5": 0.041667, + "recallAt10": 0.041667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.128116, + "directRecallAt10": 0, + "supportingRecallAt10": 0.041667 + } + }, + "qarinahV2": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.041667, + "recallAt5": 0.041667, + "recallAt10": 0.041667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.128116, + "directRecallAt10": 0, + "supportingRecallAt10": 0.041667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.041667, + "recallAt5": 0.041667, + "recallAt10": 0.041667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.128116, + "directRecallAt10": 0, + "supportingRecallAt10": 0.041667 + } + }, + "oracle": { + "deliveredTokens": 945, + "metrics": { + "recallAt1": 0.041667, + "recallAt5": 0.083333, + "recallAt10": 0.083333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.465181, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.083333 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1989, + "metrics": { + "recallAt1": 0.041667, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.434671, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.125 + } + }, + "balancedV1": { + "deliveredTokens": 1960, + "metrics": { + "recallAt1": 0.041667, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.328183, + "directRecallAt10": 0, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2": { + "deliveredTokens": 1989, + "metrics": { + "recallAt1": 0.041667, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.434671, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.125 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1989, + "metrics": { + "recallAt1": 0.041667, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.434671, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.125 + } + }, + "oracle": { + "deliveredTokens": 1977, + "metrics": { + "recallAt1": 0.041667, + "recallAt5": 0.208333, + "recallAt10": 0.208333, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.795642, + "directRecallAt10": 1, + "supportingRecallAt10": 0.208333 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3988, + "metrics": { + "recallAt1": 0.041667, + "recallAt5": 0.208333, + "recallAt10": 0.333333, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.668167, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 3988, + "metrics": { + "recallAt1": 0.041667, + "recallAt5": 0.208333, + "recallAt10": 0.333333, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.616856, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 3988, + "metrics": { + "recallAt1": 0.041667, + "recallAt5": 0.208333, + "recallAt10": 0.333333, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.668167, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3988, + "metrics": { + "recallAt1": 0.041667, + "recallAt5": 0.208333, + "recallAt10": 0.333333, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.668167, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 3927, + "metrics": { + "recallAt1": 0.041667, + "recallAt5": 0.208333, + "recallAt10": 0.333333, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.924399, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7886, + "metrics": { + "recallAt1": 0.041667, + "recallAt5": 0.208333, + "recallAt10": 0.375, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.783867, + "directRecallAt10": 1, + "supportingRecallAt10": 0.375 + } + }, + "balancedV1": { + "deliveredTokens": 7917, + "metrics": { + "recallAt1": 0.041667, + "recallAt5": 0.208333, + "recallAt10": 0.416667, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.777867, + "directRecallAt10": 1, + "supportingRecallAt10": 0.416667 + } + }, + "qarinahV2": { + "deliveredTokens": 7886, + "metrics": { + "recallAt1": 0.041667, + "recallAt5": 0.208333, + "recallAt10": 0.375, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.783867, + "directRecallAt10": 1, + "supportingRecallAt10": 0.375 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7886, + "metrics": { + "recallAt1": 0.041667, + "recallAt5": 0.208333, + "recallAt10": 0.375, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.783867, + "directRecallAt10": 1, + "supportingRecallAt10": 0.375 + } + }, + "oracle": { + "deliveredTokens": 7932, + "metrics": { + "recallAt1": 0.041667, + "recallAt5": 0.208333, + "recallAt10": 0.416667, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.416667 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.503846, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13220", + "repositorySequence": 44, + "positiveUnderStructuralOracle": true, + "directRecords": 2, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.4, + "recallAt10": 0.6, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.535152, + "directRecallAt10": 1, + "supportingRecallAt10": 0.6 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.4, + "recallAt10": 0.6, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.535152, + "directRecallAt10": 1, + "supportingRecallAt10": 0.6 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.4, + "recallAt10": 0.6, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.535152, + "directRecallAt10": 1, + "supportingRecallAt10": 0.6 + }, + "oracle": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4818, + "balancedV1": 4112, + "qarinahV2": 4818, + "qarinahV2NoGraph": 4818, + "oracle": 1941 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 512, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 488, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.262616, + "directRecallAt10": 0, + "supportingRecallAt10": 0.4 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 943, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.304781, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.2 + } + }, + "balancedV1": { + "deliveredTokens": 909, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 943, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.304781, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 943, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.304781, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.2 + } + }, + "oracle": { + "deliveredTokens": 903, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.584661, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.4 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1950, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.37413, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.4 + } + }, + "balancedV1": { + "deliveredTokens": 1971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1950, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.37413, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1950, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.37413, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.4 + } + }, + "oracle": { + "deliveredTokens": 1941, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.4, + "recallAt10": 0.6, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.535152, + "directRecallAt10": 1, + "supportingRecallAt10": 0.6 + } + }, + "balancedV1": { + "deliveredTokens": 3981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.4, + "recallAt10": 0.6, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.535152, + "directRecallAt10": 1, + "supportingRecallAt10": 0.6 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.4, + "recallAt10": 0.6, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.535152, + "directRecallAt10": 1, + "supportingRecallAt10": 0.6 + } + }, + "oracle": { + "deliveredTokens": 1941, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7976, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.4, + "recallAt10": 0.6, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.535152, + "directRecallAt10": 1, + "supportingRecallAt10": 0.6 + } + }, + "balancedV1": { + "deliveredTokens": 7924, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7976, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.4, + "recallAt10": 0.6, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.535152, + "directRecallAt10": 1, + "supportingRecallAt10": 0.6 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7976, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.4, + "recallAt10": 0.6, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.535152, + "directRecallAt10": 1, + "supportingRecallAt10": 0.6 + } + }, + "oracle": { + "deliveredTokens": 1941, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.468868, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13230", + "repositorySequence": 45, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 4, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.459055, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.459055, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.459055, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + }, + "oracle": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3428, + "balancedV1": 3706, + "qarinahV2": 3428, + "qarinahV2NoGraph": 3428, + "oracle": 1091 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.246302, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.246302, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.246302, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 427, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 909, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.19519, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 988, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 909, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.19519, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 909, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.19519, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 716, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1920, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.34621, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 1983, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1920, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.34621, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1920, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.34621, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 1091, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3868, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.459055, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "balancedV1": { + "deliveredTokens": 3893, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3868, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.459055, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3868, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.459055, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "oracle": { + "deliveredTokens": 1091, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7879, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.459055, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "balancedV1": { + "deliveredTokens": 7935, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7879, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.459055, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7879, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.459055, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "oracle": { + "deliveredTokens": 1091, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.355, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "NO_CODE_ENTITY_MATCH", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13265", + "repositorySequence": 46, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 24, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.04, + "recallAt5": 0.2, + "recallAt10": 0.36, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.838986, + "directRecallAt10": 1, + "supportingRecallAt10": 0.36 + }, + "balancedV1": { + "recallAt1": 0.04, + "recallAt5": 0.2, + "recallAt10": 0.4, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.694356, + "directRecallAt10": 0, + "supportingRecallAt10": 0.4 + }, + "qarinahV2": { + "recallAt1": 0.04, + "recallAt5": 0.2, + "recallAt10": 0.36, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.838986, + "directRecallAt10": 1, + "supportingRecallAt10": 0.36 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.04, + "recallAt5": 0.2, + "recallAt10": 0.36, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.838986, + "directRecallAt10": 1, + "supportingRecallAt10": 0.36 + }, + "oracle": { + "recallAt1": 0.04, + "recallAt5": 0.2, + "recallAt10": 0.4, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.4 + } + }, + "volumes": { + "bm25Admitted": 4947, + "balancedV1": 5085, + "qarinahV2": 4947, + "qarinahV2NoGraph": 4947, + "oracle": 4563 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 455, + "metrics": { + "recallAt1": 0.04, + "recallAt5": 0.04, + "recallAt10": 0.04, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.04 + } + }, + "balancedV1": { + "deliveredTokens": 455, + "metrics": { + "recallAt1": 0.04, + "recallAt5": 0.04, + "recallAt10": 0.04, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.04 + } + }, + "qarinahV2": { + "deliveredTokens": 455, + "metrics": { + "recallAt1": 0.04, + "recallAt5": 0.04, + "recallAt10": 0.04, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.04 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 455, + "metrics": { + "recallAt1": 0.04, + "recallAt5": 0.04, + "recallAt10": 0.04, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.04 + } + }, + "oracle": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.04, + "recallAt5": 0.08, + "recallAt10": 0.08, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.08 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 955, + "metrics": { + "recallAt1": 0.04, + "recallAt5": 0.12, + "recallAt10": 0.12, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.518493, + "directRecallAt10": 1, + "supportingRecallAt10": 0.12 + } + }, + "balancedV1": { + "deliveredTokens": 914, + "metrics": { + "recallAt1": 0.04, + "recallAt5": 0.08, + "recallAt10": 0.08, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.08 + } + }, + "qarinahV2": { + "deliveredTokens": 955, + "metrics": { + "recallAt1": 0.04, + "recallAt5": 0.12, + "recallAt10": 0.12, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.518493, + "directRecallAt10": 1, + "supportingRecallAt10": 0.12 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 955, + "metrics": { + "recallAt1": 0.04, + "recallAt5": 0.12, + "recallAt10": 0.12, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.518493, + "directRecallAt10": 1, + "supportingRecallAt10": 0.12 + } + }, + "oracle": { + "deliveredTokens": 901, + "metrics": { + "recallAt1": 0.04, + "recallAt5": 0.16, + "recallAt10": 0.16, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.697114, + "directRecallAt10": 1, + "supportingRecallAt10": 0.16 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1900, + "metrics": { + "recallAt1": 0.04, + "recallAt5": 0.16, + "recallAt10": 0.16, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.58431, + "directRecallAt10": 1, + "supportingRecallAt10": 0.16 + } + }, + "balancedV1": { + "deliveredTokens": 1950, + "metrics": { + "recallAt1": 0.04, + "recallAt5": 0.16, + "recallAt10": 0.16, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.39147, + "directRecallAt10": 0, + "supportingRecallAt10": 0.16 + } + }, + "qarinahV2": { + "deliveredTokens": 1900, + "metrics": { + "recallAt1": 0.04, + "recallAt5": 0.16, + "recallAt10": 0.16, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.58431, + "directRecallAt10": 1, + "supportingRecallAt10": 0.16 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1900, + "metrics": { + "recallAt1": 0.04, + "recallAt5": 0.16, + "recallAt10": 0.16, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.58431, + "directRecallAt10": 1, + "supportingRecallAt10": 0.16 + } + }, + "oracle": { + "deliveredTokens": 1975, + "metrics": { + "recallAt1": 0.04, + "recallAt5": 0.2, + "recallAt10": 0.28, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.861611, + "directRecallAt10": 1, + "supportingRecallAt10": 0.28 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3974, + "metrics": { + "recallAt1": 0.04, + "recallAt5": 0.2, + "recallAt10": 0.28, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.748806, + "directRecallAt10": 1, + "supportingRecallAt10": 0.28 + } + }, + "balancedV1": { + "deliveredTokens": 3965, + "metrics": { + "recallAt1": 0.04, + "recallAt5": 0.2, + "recallAt10": 0.28, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.555966, + "directRecallAt10": 0, + "supportingRecallAt10": 0.28 + } + }, + "qarinahV2": { + "deliveredTokens": 3974, + "metrics": { + "recallAt1": 0.04, + "recallAt5": 0.2, + "recallAt10": 0.28, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.748806, + "directRecallAt10": 1, + "supportingRecallAt10": 0.28 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3974, + "metrics": { + "recallAt1": 0.04, + "recallAt5": 0.2, + "recallAt10": 0.28, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.748806, + "directRecallAt10": 1, + "supportingRecallAt10": 0.28 + } + }, + "oracle": { + "deliveredTokens": 3896, + "metrics": { + "recallAt1": 0.04, + "recallAt5": 0.2, + "recallAt10": 0.4, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.4 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7942, + "metrics": { + "recallAt1": 0.04, + "recallAt5": 0.2, + "recallAt10": 0.36, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.838986, + "directRecallAt10": 1, + "supportingRecallAt10": 0.36 + } + }, + "balancedV1": { + "deliveredTokens": 7941, + "metrics": { + "recallAt1": 0.04, + "recallAt5": 0.2, + "recallAt10": 0.4, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.694356, + "directRecallAt10": 0, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2": { + "deliveredTokens": 7942, + "metrics": { + "recallAt1": 0.04, + "recallAt5": 0.2, + "recallAt10": 0.36, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.838986, + "directRecallAt10": 1, + "supportingRecallAt10": 0.36 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7942, + "metrics": { + "recallAt1": 0.04, + "recallAt5": 0.2, + "recallAt10": 0.36, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.838986, + "directRecallAt10": 1, + "supportingRecallAt10": 0.36 + } + }, + "oracle": { + "deliveredTokens": 7960, + "metrics": { + "recallAt1": 0.04, + "recallAt5": 0.2, + "recallAt10": 0.4, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.4 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.492399, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13315", + "repositorySequence": 47, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.315465, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.315465, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.315465, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4875, + "balancedV1": 4877, + "qarinahV2": 4875, + "qarinahV2NoGraph": 4875, + "oracle": 515 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 382, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 455, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 382, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 382, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 0, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 980, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 946, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 980, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 980, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3903, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.315465, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3924, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3903, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.315465, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3903, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.315465, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.315465, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 7941, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.315465, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.315465, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 515, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.525, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 9, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13321", + "repositorySequence": 48, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 5, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.4, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.282634, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.4, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.282634, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.4, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.282634, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + }, + "oracle": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 5902, + "balancedV1": 5005, + "qarinahV2": 5902, + "qarinahV2NoGraph": 5902, + "oracle": 1234 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 511, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 511, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 511, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 427, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.553146, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 880, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.213986, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "balancedV1": { + "deliveredTokens": 991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 880, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.213986, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 880, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.213986, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "oracle": { + "deliveredTokens": 859, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.8, + "recallAt10": 0.8, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.868795, + "directRecallAt10": null, + "supportingRecallAt10": 0.8 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1963, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.16958, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "balancedV1": { + "deliveredTokens": 1985, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1963, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.16958, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1963, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.16958, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "oracle": { + "deliveredTokens": 1234, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.16958, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "balancedV1": { + "deliveredTokens": 3948, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.16958, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.16958, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "oracle": { + "deliveredTokens": 1234, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7961, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.4, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.282634, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + }, + "balancedV1": { + "deliveredTokens": 7992, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7961, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.4, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.282634, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7961, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.4, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.282634, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + }, + "oracle": { + "deliveredTokens": 1234, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.386951, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13401", + "repositorySequence": 49, + "positiveUnderStructuralOracle": true, + "directRecords": 6, + "supportingRecords": 20, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.038462, + "recallAt5": 0.153846, + "recallAt10": 0.307692, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.38147, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.307692 + }, + "balancedV1": { + "recallAt1": 0.038462, + "recallAt5": 0.192308, + "recallAt10": 0.384615, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.608663, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.384615 + }, + "qarinahV2": { + "recallAt1": 0.038462, + "recallAt5": 0.153846, + "recallAt10": 0.307692, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.38147, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.307692 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.038462, + "recallAt5": 0.153846, + "recallAt10": 0.307692, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.38147, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.307692 + }, + "oracle": { + "recallAt1": 0.038462, + "recallAt5": 0.192308, + "recallAt10": 0.384615, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.384615 + } + }, + "volumes": { + "bm25Admitted": 4995, + "balancedV1": 4790, + "qarinahV2": 4995, + "qarinahV2NoGraph": 4995, + "oracle": 3451 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 435, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.038462, + "recallAt10": 0.038462, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.056571, + "directRecallAt10": 0, + "supportingRecallAt10": 0.038462 + } + }, + "balancedV1": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.038462, + "recallAt5": 0.076923, + "recallAt10": 0.076923, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.438701, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.076923 + } + }, + "qarinahV2": { + "deliveredTokens": 435, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.038462, + "recallAt10": 0.038462, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.056571, + "directRecallAt10": 0, + "supportingRecallAt10": 0.038462 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 435, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.038462, + "recallAt10": 0.038462, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.056571, + "directRecallAt10": 0, + "supportingRecallAt10": 0.038462 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.038462, + "recallAt5": 0.076923, + "recallAt10": 0.076923, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.438701, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.076923 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.038462, + "recallAt5": 0.038462, + "recallAt10": 0.038462, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.089663, + "directRecallAt10": 0, + "supportingRecallAt10": 0.038462 + } + }, + "balancedV1": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.038462, + "recallAt5": 0.038462, + "recallAt10": 0.038462, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.089663, + "directRecallAt10": 0, + "supportingRecallAt10": 0.038462 + } + }, + "qarinahV2": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.038462, + "recallAt5": 0.038462, + "recallAt10": 0.038462, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.089663, + "directRecallAt10": 0, + "supportingRecallAt10": 0.038462 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.038462, + "recallAt5": 0.038462, + "recallAt10": 0.038462, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.089663, + "directRecallAt10": 0, + "supportingRecallAt10": 0.038462 + } + }, + "oracle": { + "deliveredTokens": 990, + "metrics": { + "recallAt1": 0.038462, + "recallAt5": 0.153846, + "recallAt10": 0.153846, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.689043, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.153846 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1968, + "metrics": { + "recallAt1": 0.038462, + "recallAt5": 0.115385, + "recallAt10": 0.115385, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.250341, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.115385 + } + }, + "balancedV1": { + "deliveredTokens": 1965, + "metrics": { + "recallAt1": 0.038462, + "recallAt5": 0.115385, + "recallAt10": 0.115385, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.280728, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.115385 + } + }, + "qarinahV2": { + "deliveredTokens": 1968, + "metrics": { + "recallAt1": 0.038462, + "recallAt5": 0.115385, + "recallAt10": 0.115385, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.250341, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.115385 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1968, + "metrics": { + "recallAt1": 0.038462, + "recallAt5": 0.115385, + "recallAt10": 0.115385, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.250341, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.115385 + } + }, + "oracle": { + "deliveredTokens": 1920, + "metrics": { + "recallAt1": 0.038462, + "recallAt5": 0.192308, + "recallAt10": 0.269231, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.918805, + "directRecallAt10": 1, + "supportingRecallAt10": 0.269231 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3902, + "metrics": { + "recallAt1": 0.038462, + "recallAt5": 0.153846, + "recallAt10": 0.269231, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.354479, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.269231 + } + }, + "balancedV1": { + "deliveredTokens": 3876, + "metrics": { + "recallAt1": 0.038462, + "recallAt5": 0.192308, + "recallAt10": 0.307692, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.560488, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.307692 + } + }, + "qarinahV2": { + "deliveredTokens": 3902, + "metrics": { + "recallAt1": 0.038462, + "recallAt5": 0.153846, + "recallAt10": 0.269231, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.354479, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.269231 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3902, + "metrics": { + "recallAt1": 0.038462, + "recallAt5": 0.153846, + "recallAt10": 0.269231, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.354479, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.269231 + } + }, + "oracle": { + "deliveredTokens": 3996, + "metrics": { + "recallAt1": 0.038462, + "recallAt5": 0.192308, + "recallAt10": 0.384615, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.384615 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7900, + "metrics": { + "recallAt1": 0.038462, + "recallAt5": 0.153846, + "recallAt10": 0.307692, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.38147, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.307692 + } + }, + "balancedV1": { + "deliveredTokens": 7917, + "metrics": { + "recallAt1": 0.038462, + "recallAt5": 0.192308, + "recallAt10": 0.384615, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.608663, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.384615 + } + }, + "qarinahV2": { + "deliveredTokens": 7900, + "metrics": { + "recallAt1": 0.038462, + "recallAt5": 0.153846, + "recallAt10": 0.307692, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.38147, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.307692 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7900, + "metrics": { + "recallAt1": 0.038462, + "recallAt5": 0.153846, + "recallAt10": 0.307692, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.38147, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.307692 + } + }, + "oracle": { + "deliveredTokens": 7802, + "metrics": { + "recallAt1": 0.038462, + "recallAt5": 0.192308, + "recallAt10": 0.384615, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.384615 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.559286, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13447", + "repositorySequence": 50, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 6, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.166667, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.100867, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.166667, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.100867, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.166667, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.100867, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + }, + "oracle": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3975, + "balancedV1": 4942, + "qarinahV2": 3975, + "qarinahV2NoGraph": 3975, + "oracle": 2394 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 471, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.190921, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "balancedV1": { + "deliveredTokens": 512, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 471, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.190921, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 471, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.190921, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "oracle": { + "deliveredTokens": 427, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.493523, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 978, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.130324, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "balancedV1": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 978, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.130324, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 978, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.130324, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "oracle": { + "deliveredTokens": 859, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.775148, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1940, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1966, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1940, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1940, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 1876, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.775148, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3975, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.166667, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.100867, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "balancedV1": { + "deliveredTokens": 3989, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3975, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.166667, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.100867, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3975, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.166667, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.100867, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "oracle": { + "deliveredTokens": 2394, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7946, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.166667, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.100867, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "balancedV1": { + "deliveredTokens": 7884, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7946, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.166667, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.100867, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7946, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.166667, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.100867, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "oracle": { + "deliveredTokens": 2394, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.5125, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13448", + "repositorySequence": 51, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 26, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.037037, + "recallAt5": 0.111111, + "recallAt10": 0.185185, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.506337, + "directRecallAt10": 1, + "supportingRecallAt10": 0.185185 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.148148, + "recallAt10": 0.296296, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.49553, + "directRecallAt10": 0, + "supportingRecallAt10": 0.296296 + }, + "qarinahV2": { + "recallAt1": 0.037037, + "recallAt5": 0.111111, + "recallAt10": 0.185185, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.506337, + "directRecallAt10": 1, + "supportingRecallAt10": 0.185185 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.037037, + "recallAt5": 0.111111, + "recallAt10": 0.185185, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.506337, + "directRecallAt10": 1, + "supportingRecallAt10": 0.185185 + }, + "oracle": { + "recallAt1": 0.037037, + "recallAt5": 0.185185, + "recallAt10": 0.37037, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.37037 + } + }, + "volumes": { + "bm25Admitted": 7055, + "balancedV1": 6145, + "qarinahV2": 7055, + "qarinahV2NoGraph": 7055, + "oracle": 4550 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 511, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.037037, + "recallAt5": 0.037037, + "recallAt10": 0.037037, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.037037 + } + }, + "qarinahV2": { + "deliveredTokens": 511, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 511, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 408, + "metrics": { + "recallAt1": 0.037037, + "recallAt5": 0.074074, + "recallAt10": 0.074074, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.074074 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 969, + "metrics": { + "recallAt1": 0.037037, + "recallAt5": 0.074074, + "recallAt10": 0.074074, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.074074 + } + }, + "balancedV1": { + "deliveredTokens": 991, + "metrics": { + "recallAt1": 0.037037, + "recallAt5": 0.074074, + "recallAt10": 0.074074, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.074074 + } + }, + "qarinahV2": { + "deliveredTokens": 969, + "metrics": { + "recallAt1": 0.037037, + "recallAt5": 0.074074, + "recallAt10": 0.074074, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.074074 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 969, + "metrics": { + "recallAt1": 0.037037, + "recallAt5": 0.074074, + "recallAt10": 0.074074, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.074074 + } + }, + "oracle": { + "deliveredTokens": 969, + "metrics": { + "recallAt1": 0.037037, + "recallAt5": 0.074074, + "recallAt10": 0.074074, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.074074 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1993, + "metrics": { + "recallAt1": 0.037037, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.29505, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + } + }, + "balancedV1": { + "deliveredTokens": 1922, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.037037, + "recallAt10": 0.037037, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.09642, + "directRecallAt10": 0, + "supportingRecallAt10": 0.037037 + } + }, + "qarinahV2": { + "deliveredTokens": 1993, + "metrics": { + "recallAt1": 0.037037, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.29505, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1993, + "metrics": { + "recallAt1": 0.037037, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.29505, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + } + }, + "oracle": { + "deliveredTokens": 1990, + "metrics": { + "recallAt1": 0.037037, + "recallAt5": 0.185185, + "recallAt10": 0.185185, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.756234, + "directRecallAt10": 1, + "supportingRecallAt10": 0.185185 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3947, + "metrics": { + "recallAt1": 0.037037, + "recallAt5": 0.037037, + "recallAt10": 0.037037, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.037037 + } + }, + "balancedV1": { + "deliveredTokens": 3989, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.148148, + "recallAt10": 0.185185, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.352204, + "directRecallAt10": 0, + "supportingRecallAt10": 0.185185 + } + }, + "qarinahV2": { + "deliveredTokens": 3947, + "metrics": { + "recallAt1": 0.037037, + "recallAt5": 0.037037, + "recallAt10": 0.037037, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.037037 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3947, + "metrics": { + "recallAt1": 0.037037, + "recallAt5": 0.037037, + "recallAt10": 0.037037, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.037037 + } + }, + "oracle": { + "deliveredTokens": 3874, + "metrics": { + "recallAt1": 0.037037, + "recallAt5": 0.185185, + "recallAt10": 0.333333, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.955825, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7958, + "metrics": { + "recallAt1": 0.037037, + "recallAt5": 0.111111, + "recallAt10": 0.185185, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.506337, + "directRecallAt10": 1, + "supportingRecallAt10": 0.185185 + } + }, + "balancedV1": { + "deliveredTokens": 7993, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.148148, + "recallAt10": 0.296296, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.49553, + "directRecallAt10": 0, + "supportingRecallAt10": 0.296296 + } + }, + "qarinahV2": { + "deliveredTokens": 7958, + "metrics": { + "recallAt1": 0.037037, + "recallAt5": 0.111111, + "recallAt10": 0.185185, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.506337, + "directRecallAt10": 1, + "supportingRecallAt10": 0.185185 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7958, + "metrics": { + "recallAt1": 0.037037, + "recallAt5": 0.111111, + "recallAt10": 0.185185, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.506337, + "directRecallAt10": 1, + "supportingRecallAt10": 0.185185 + } + }, + "oracle": { + "deliveredTokens": 7968, + "metrics": { + "recallAt1": 0.037037, + "recallAt5": 0.185185, + "recallAt10": 0.37037, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.37037 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.487056, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13551", + "repositorySequence": 52, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 7, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.285714, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.174371, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.285714, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.174371, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.285714, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.174371, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + }, + "oracle": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2641, + "balancedV1": 2631, + "qarinahV2": 2641, + "qarinahV2NoGraph": 2641, + "oracle": 2545 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 477, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 433, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 477, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 477, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 427, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.448304, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 910, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 867, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.571429, + "recallAt10": 0.571429, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.704125, + "directRecallAt10": null, + "supportingRecallAt10": 0.571429 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.142857, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.091625, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "balancedV1": { + "deliveredTokens": 1928, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.142857, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.091625, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.142857, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.091625, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "oracle": { + "deliveredTokens": 1876, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.571429, + "recallAt10": 0.571429, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.704125, + "directRecallAt10": null, + "supportingRecallAt10": 0.571429 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.285714, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.174371, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + }, + "balancedV1": { + "deliveredTokens": 3996, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.285714, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.174371, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.285714, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.174371, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + }, + "oracle": { + "deliveredTokens": 2545, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7988, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.285714, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.174371, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + }, + "balancedV1": { + "deliveredTokens": 7922, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7988, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.285714, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.174371, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7988, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.285714, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.174371, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + }, + "oracle": { + "deliveredTokens": 2545, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.22, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "NO_CODE_ENTITY_MATCH", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13590", + "repositorySequence": 53, + "positiveUnderStructuralOracle": true, + "directRecords": 3, + "supportingRecords": 25, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.285714, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.597775, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.285714 + }, + "balancedV1": { + "recallAt1": 0.035714, + "recallAt5": 0.178571, + "recallAt10": 0.357143, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.830996, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.357143 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.285714, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.597775, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.285714 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.285714, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.597775, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.285714 + }, + "oracle": { + "recallAt1": 0.035714, + "recallAt5": 0.178571, + "recallAt10": 0.357143, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.357143 + } + }, + "volumes": { + "bm25Admitted": 5631, + "balancedV1": 6307, + "qarinahV2": 5631, + "qarinahV2NoGraph": 5631, + "oracle": 4928 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 495, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.035714, + "recallAt10": 0.035714, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.071652, + "directRecallAt10": 0, + "supportingRecallAt10": 0.035714 + } + }, + "balancedV1": { + "deliveredTokens": 510, + "metrics": { + "recallAt1": 0.035714, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.412352, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.071429 + } + }, + "qarinahV2": { + "deliveredTokens": 495, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.035714, + "recallAt10": 0.035714, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.071652, + "directRecallAt10": 0, + "supportingRecallAt10": 0.035714 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 495, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.035714, + "recallAt10": 0.035714, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.071652, + "directRecallAt10": 0, + "supportingRecallAt10": 0.035714 + } + }, + "oracle": { + "deliveredTokens": 484, + "metrics": { + "recallAt1": 0.035714, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.412352, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.071429 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 875, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.271741, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.071429 + } + }, + "balancedV1": { + "deliveredTokens": 969, + "metrics": { + "recallAt1": 0.035714, + "recallAt5": 0.107143, + "recallAt10": 0.107143, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469135, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.107143 + } + }, + "qarinahV2": { + "deliveredTokens": 875, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.271741, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.071429 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 875, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.271741, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.071429 + } + }, + "oracle": { + "deliveredTokens": 955, + "metrics": { + "recallAt1": 0.035714, + "recallAt5": 0.107143, + "recallAt10": 0.107143, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469135, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.107143 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1943, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.271741, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.071429 + } + }, + "balancedV1": { + "deliveredTokens": 1967, + "metrics": { + "recallAt1": 0.035714, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.518045, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2": { + "deliveredTokens": 1943, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.271741, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.071429 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1943, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.271741, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.071429 + } + }, + "oracle": { + "deliveredTokens": 1962, + "metrics": { + "recallAt1": 0.035714, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.774917, + "directRecallAt10": 1, + "supportingRecallAt10": 0.142857 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3916, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.178571, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.492905, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.178571 + } + }, + "balancedV1": { + "deliveredTokens": 3996, + "metrics": { + "recallAt1": 0.035714, + "recallAt5": 0.178571, + "recallAt10": 0.214286, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.690299, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.214286 + } + }, + "qarinahV2": { + "deliveredTokens": 3916, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.178571, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.492905, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.178571 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3916, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.178571, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.492905, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.178571 + } + }, + "oracle": { + "deliveredTokens": 3930, + "metrics": { + "recallAt1": 0.035714, + "recallAt5": 0.178571, + "recallAt10": 0.321429, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.967172, + "directRecallAt10": 1, + "supportingRecallAt10": 0.321429 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7947, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.285714, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.597775, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.285714 + } + }, + "balancedV1": { + "deliveredTokens": 7979, + "metrics": { + "recallAt1": 0.035714, + "recallAt5": 0.178571, + "recallAt10": 0.357143, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.830996, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.357143 + } + }, + "qarinahV2": { + "deliveredTokens": 7947, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.285714, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.597775, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.285714 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7947, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.285714, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.597775, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.285714 + } + }, + "oracle": { + "deliveredTokens": 7968, + "metrics": { + "recallAt1": 0.035714, + "recallAt5": 0.178571, + "recallAt10": 0.357143, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.357143 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.619531, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13658", + "repositorySequence": 54, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 4, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.25, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.112845, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.25, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.112845, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.25, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.112845, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + }, + "oracle": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 6013, + "balancedV1": 5721, + "qarinahV2": 6013, + "qarinahV2NoGraph": 6013, + "oracle": 996 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 495, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.246302, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 455, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 495, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.246302, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 495, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.246302, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 488, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 954, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.19519, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 914, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 954, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.19519, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 954, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.19519, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 996, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1955, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1950, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1955, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1955, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 996, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 4000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.25, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.139056, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 3982, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 4000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.25, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.139056, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 4000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.25, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.139056, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 996, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.25, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.112845, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 7870, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.25, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.112845, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.25, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.112845, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 996, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.577778, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13660", + "repositorySequence": 55, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 5, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.213986, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.213986, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.213986, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + }, + "oracle": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 7206, + "balancedV1": 6308, + "qarinahV2": 7206, + "qarinahV2NoGraph": 7206, + "oracle": 1371 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.33916, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "balancedV1": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.33916, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.33916, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "oracle": { + "deliveredTokens": 375, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.33916, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 980, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.33916, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "balancedV1": { + "deliveredTokens": 985, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 980, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.33916, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 980, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.33916, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "oracle": { + "deliveredTokens": 863, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.6, + "recallAt10": 0.6, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.722727, + "directRecallAt10": null, + "supportingRecallAt10": 0.6 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1905, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.213986, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "balancedV1": { + "deliveredTokens": 1954, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1905, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.213986, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1905, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.213986, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "oracle": { + "deliveredTokens": 1371, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.213986, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "balancedV1": { + "deliveredTokens": 3986, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.213986, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.213986, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "oracle": { + "deliveredTokens": 1371, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7985, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.213986, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "balancedV1": { + "deliveredTokens": 7899, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7985, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.213986, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7985, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.213986, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "oracle": { + "deliveredTokens": 1371, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.603153, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13710", + "repositorySequence": 56, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 8, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.222222, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.118804, + "directRecallAt10": 0, + "supportingRecallAt10": 0.222222 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.222222, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.118804, + "directRecallAt10": 0, + "supportingRecallAt10": 0.222222 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.222222, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.118804, + "directRecallAt10": 0, + "supportingRecallAt10": 0.222222 + }, + "oracle": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3789, + "balancedV1": 4918, + "qarinahV2": 3789, + "qarinahV2NoGraph": 3789, + "oracle": 3376 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 504, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.100876, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + } + }, + "balancedV1": { + "deliveredTokens": 484, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 504, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.100876, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 504, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.100876, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + } + }, + "oracle": { + "deliveredTokens": 393, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.260761, + "directRecallAt10": 0, + "supportingRecallAt10": 0.222222 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 937, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 943, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 937, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 937, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 941, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.660474, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.2, + "ndcgAt10": 0.172099, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.2, + "ndcgAt10": 0.172099, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.2, + "ndcgAt10": 0.172099, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 1950, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.660474, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3965, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.222222, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.118804, + "directRecallAt10": 0, + "supportingRecallAt10": 0.222222 + } + }, + "balancedV1": { + "deliveredTokens": 3989, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3965, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.222222, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.118804, + "directRecallAt10": 0, + "supportingRecallAt10": 0.222222 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3965, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.222222, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.118804, + "directRecallAt10": 0, + "supportingRecallAt10": 0.222222 + } + }, + "oracle": { + "deliveredTokens": 3376, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7932, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.222222, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.118804, + "directRecallAt10": 0, + "supportingRecallAt10": 0.222222 + } + }, + "balancedV1": { + "deliveredTokens": 7913, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7932, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.222222, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.118804, + "directRecallAt10": 0, + "supportingRecallAt10": 0.222222 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7932, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.222222, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.118804, + "directRecallAt10": 0, + "supportingRecallAt10": 0.222222 + } + }, + "oracle": { + "deliveredTokens": 3376, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.524107, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13757", + "repositorySequence": 57, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 28, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.034483, + "recallAt5": 0.172414, + "recallAt10": 0.310345, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.63992, + "directRecallAt10": 0, + "supportingRecallAt10": 0.310345 + }, + "balancedV1": { + "recallAt1": 0.034483, + "recallAt5": 0.172414, + "recallAt10": 0.344828, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.790776, + "directRecallAt10": 1, + "supportingRecallAt10": 0.344828 + }, + "qarinahV2": { + "recallAt1": 0.034483, + "recallAt5": 0.172414, + "recallAt10": 0.310345, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.63992, + "directRecallAt10": 0, + "supportingRecallAt10": 0.310345 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.034483, + "recallAt5": 0.172414, + "recallAt10": 0.310345, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.63992, + "directRecallAt10": 0, + "supportingRecallAt10": 0.310345 + }, + "oracle": { + "recallAt1": 0.034483, + "recallAt5": 0.172414, + "recallAt10": 0.344828, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.344828 + } + }, + "volumes": { + "bm25Admitted": 5804, + "balancedV1": 5543, + "qarinahV2": 5804, + "qarinahV2NoGraph": 5804, + "oracle": 4016 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 408, + "metrics": { + "recallAt1": 0.034483, + "recallAt5": 0.034483, + "recallAt10": 0.034483, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.034483 + } + }, + "balancedV1": { + "deliveredTokens": 510, + "metrics": { + "recallAt1": 0.034483, + "recallAt5": 0.068966, + "recallAt10": 0.068966, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.068966 + } + }, + "qarinahV2": { + "deliveredTokens": 408, + "metrics": { + "recallAt1": 0.034483, + "recallAt5": 0.034483, + "recallAt10": 0.034483, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.034483 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 408, + "metrics": { + "recallAt1": 0.034483, + "recallAt5": 0.034483, + "recallAt10": 0.034483, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.034483 + } + }, + "oracle": { + "deliveredTokens": 484, + "metrics": { + "recallAt1": 0.034483, + "recallAt5": 0.068966, + "recallAt10": 0.068966, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.068966 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 931, + "metrics": { + "recallAt1": 0.034483, + "recallAt5": 0.068966, + "recallAt10": 0.068966, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.068966 + } + }, + "balancedV1": { + "deliveredTokens": 969, + "metrics": { + "recallAt1": 0.034483, + "recallAt5": 0.103448, + "recallAt10": 0.103448, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.325653, + "directRecallAt10": 0, + "supportingRecallAt10": 0.103448 + } + }, + "qarinahV2": { + "deliveredTokens": 931, + "metrics": { + "recallAt1": 0.034483, + "recallAt5": 0.068966, + "recallAt10": 0.068966, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.068966 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 931, + "metrics": { + "recallAt1": 0.034483, + "recallAt5": 0.068966, + "recallAt10": 0.068966, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.068966 + } + }, + "oracle": { + "deliveredTokens": 980, + "metrics": { + "recallAt1": 0.034483, + "recallAt5": 0.137931, + "recallAt10": 0.137931, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.697114, + "directRecallAt10": 1, + "supportingRecallAt10": 0.137931 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1929, + "metrics": { + "recallAt1": 0.034483, + "recallAt5": 0.103448, + "recallAt10": 0.103448, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.325653, + "directRecallAt10": 0, + "supportingRecallAt10": 0.103448 + } + }, + "balancedV1": { + "deliveredTokens": 1948, + "metrics": { + "recallAt1": 0.034483, + "recallAt5": 0.137931, + "recallAt10": 0.137931, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.39147, + "directRecallAt10": 0, + "supportingRecallAt10": 0.137931 + } + }, + "qarinahV2": { + "deliveredTokens": 1929, + "metrics": { + "recallAt1": 0.034483, + "recallAt5": 0.103448, + "recallAt10": 0.103448, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.325653, + "directRecallAt10": 0, + "supportingRecallAt10": 0.103448 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1929, + "metrics": { + "recallAt1": 0.034483, + "recallAt5": 0.103448, + "recallAt10": 0.103448, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.325653, + "directRecallAt10": 0, + "supportingRecallAt10": 0.103448 + } + }, + "oracle": { + "deliveredTokens": 1963, + "metrics": { + "recallAt1": 0.034483, + "recallAt5": 0.172414, + "recallAt10": 0.241379, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.861611, + "directRecallAt10": 1, + "supportingRecallAt10": 0.241379 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3978, + "metrics": { + "recallAt1": 0.034483, + "recallAt5": 0.172414, + "recallAt10": 0.172414, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.450589, + "directRecallAt10": 0, + "supportingRecallAt10": 0.172414 + } + }, + "balancedV1": { + "deliveredTokens": 3907, + "metrics": { + "recallAt1": 0.034483, + "recallAt5": 0.172414, + "recallAt10": 0.275862, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.700596, + "directRecallAt10": 1, + "supportingRecallAt10": 0.275862 + } + }, + "qarinahV2": { + "deliveredTokens": 3978, + "metrics": { + "recallAt1": 0.034483, + "recallAt5": 0.172414, + "recallAt10": 0.172414, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.450589, + "directRecallAt10": 0, + "supportingRecallAt10": 0.172414 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3978, + "metrics": { + "recallAt1": 0.034483, + "recallAt5": 0.172414, + "recallAt10": 0.172414, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.450589, + "directRecallAt10": 0, + "supportingRecallAt10": 0.172414 + } + }, + "oracle": { + "deliveredTokens": 3999, + "metrics": { + "recallAt1": 0.034483, + "recallAt5": 0.172414, + "recallAt10": 0.344828, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.344828 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7977, + "metrics": { + "recallAt1": 0.034483, + "recallAt5": 0.172414, + "recallAt10": 0.310345, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.63992, + "directRecallAt10": 0, + "supportingRecallAt10": 0.310345 + } + }, + "balancedV1": { + "deliveredTokens": 7972, + "metrics": { + "recallAt1": 0.034483, + "recallAt5": 0.172414, + "recallAt10": 0.344828, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.790776, + "directRecallAt10": 1, + "supportingRecallAt10": 0.344828 + } + }, + "qarinahV2": { + "deliveredTokens": 7977, + "metrics": { + "recallAt1": 0.034483, + "recallAt5": 0.172414, + "recallAt10": 0.310345, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.63992, + "directRecallAt10": 0, + "supportingRecallAt10": 0.310345 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7977, + "metrics": { + "recallAt1": 0.034483, + "recallAt5": 0.172414, + "recallAt10": 0.310345, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.63992, + "directRecallAt10": 0, + "supportingRecallAt10": 0.310345 + } + }, + "oracle": { + "deliveredTokens": 7927, + "metrics": { + "recallAt1": 0.034483, + "recallAt5": 0.172414, + "recallAt10": 0.344828, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.344828 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.52, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13768", + "repositorySequence": 58, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 3682, + "balancedV1": 4592, + "qarinahV2": 3682, + "qarinahV2NoGraph": 3682, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 506, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 433, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 506, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 506, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 963, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 983, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 963, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 963, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1992, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1999, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1992, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1992, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3937, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3999, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3937, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3937, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7927, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 7919, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 7927, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7927, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.402586, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13925", + "repositorySequence": 59, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 29, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.1, + "recallAt10": 0.233333, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 0.5, + "ndcgAt10": 0.427507, + "directRecallAt10": 0, + "supportingRecallAt10": 0.233333 + }, + "balancedV1": { + "recallAt1": 0.033333, + "recallAt5": 0.166667, + "recallAt10": 0.333333, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.694356, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.1, + "recallAt10": 0.233333, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 0.5, + "ndcgAt10": 0.427507, + "directRecallAt10": 0, + "supportingRecallAt10": 0.233333 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.1, + "recallAt10": 0.233333, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 0.5, + "ndcgAt10": 0.427507, + "directRecallAt10": 0, + "supportingRecallAt10": 0.233333 + }, + "oracle": { + "recallAt1": 0.033333, + "recallAt5": 0.166667, + "recallAt10": 0.333333, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "volumes": { + "bm25Admitted": 3405, + "balancedV1": 4170, + "qarinahV2": 3405, + "qarinahV2NoGraph": 3405, + "oracle": 4031 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 472, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.033333, + "recallAt10": 0.033333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.09642, + "directRecallAt10": 0, + "supportingRecallAt10": 0.033333 + } + }, + "balancedV1": { + "deliveredTokens": 455, + "metrics": { + "recallAt1": 0.033333, + "recallAt5": 0.033333, + "recallAt10": 0.033333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.033333 + } + }, + "qarinahV2": { + "deliveredTokens": 472, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.033333, + "recallAt10": 0.033333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.09642, + "directRecallAt10": 0, + "supportingRecallAt10": 0.033333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 472, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.033333, + "recallAt10": 0.033333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.09642, + "directRecallAt10": 0, + "supportingRecallAt10": 0.033333 + } + }, + "oracle": { + "deliveredTokens": 408, + "metrics": { + "recallAt1": 0.033333, + "recallAt5": 0.066667, + "recallAt10": 0.066667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.066667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.033333, + "recallAt10": 0.033333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.09642, + "directRecallAt10": 0, + "supportingRecallAt10": 0.033333 + } + }, + "balancedV1": { + "deliveredTokens": 951, + "metrics": { + "recallAt1": 0.033333, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.325653, + "directRecallAt10": 0, + "supportingRecallAt10": 0.1 + } + }, + "qarinahV2": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.033333, + "recallAt10": 0.033333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.09642, + "directRecallAt10": 0, + "supportingRecallAt10": 0.033333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.033333, + "recallAt10": 0.033333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.09642, + "directRecallAt10": 0, + "supportingRecallAt10": 0.033333 + } + }, + "oracle": { + "deliveredTokens": 904, + "metrics": { + "recallAt1": 0.033333, + "recallAt5": 0.133333, + "recallAt10": 0.133333, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.697114, + "directRecallAt10": 1, + "supportingRecallAt10": 0.133333 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1996, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.1, + "recallAt10": 0.133333, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.286387, + "directRecallAt10": 0, + "supportingRecallAt10": 0.133333 + } + }, + "balancedV1": { + "deliveredTokens": 1949, + "metrics": { + "recallAt1": 0.033333, + "recallAt5": 0.133333, + "recallAt10": 0.133333, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.39147, + "directRecallAt10": 0, + "supportingRecallAt10": 0.133333 + } + }, + "qarinahV2": { + "deliveredTokens": 1996, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.1, + "recallAt10": 0.133333, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.286387, + "directRecallAt10": 0, + "supportingRecallAt10": 0.133333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1996, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.1, + "recallAt10": 0.133333, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.286387, + "directRecallAt10": 0, + "supportingRecallAt10": 0.133333 + } + }, + "oracle": { + "deliveredTokens": 1947, + "metrics": { + "recallAt1": 0.033333, + "recallAt5": 0.166667, + "recallAt10": 0.2, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.81067, + "directRecallAt10": 1, + "supportingRecallAt10": 0.2 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.1, + "recallAt10": 0.233333, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 0.5, + "ndcgAt10": 0.427507, + "directRecallAt10": 0, + "supportingRecallAt10": 0.233333 + } + }, + "balancedV1": { + "deliveredTokens": 3933, + "metrics": { + "recallAt1": 0.033333, + "recallAt5": 0.166667, + "recallAt10": 0.3, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.65018, + "directRecallAt10": 0, + "supportingRecallAt10": 0.3 + } + }, + "qarinahV2": { + "deliveredTokens": 3994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.1, + "recallAt10": 0.233333, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 0.5, + "ndcgAt10": 0.427507, + "directRecallAt10": 0, + "supportingRecallAt10": 0.233333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.1, + "recallAt10": 0.233333, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 0.5, + "ndcgAt10": 0.427507, + "directRecallAt10": 0, + "supportingRecallAt10": 0.233333 + } + }, + "oracle": { + "deliveredTokens": 3900, + "metrics": { + "recallAt1": 0.033333, + "recallAt5": 0.166667, + "recallAt10": 0.333333, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7925, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.1, + "recallAt10": 0.233333, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 0.5, + "ndcgAt10": 0.427507, + "directRecallAt10": 0, + "supportingRecallAt10": 0.233333 + } + }, + "balancedV1": { + "deliveredTokens": 7975, + "metrics": { + "recallAt1": 0.033333, + "recallAt5": 0.166667, + "recallAt10": 0.333333, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.694356, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 7925, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.1, + "recallAt10": 0.233333, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 0.5, + "ndcgAt10": 0.427507, + "directRecallAt10": 0, + "supportingRecallAt10": 0.233333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7925, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.1, + "recallAt10": 0.233333, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 0.5, + "ndcgAt10": 0.427507, + "directRecallAt10": 0, + "supportingRecallAt10": 0.233333 + } + }, + "oracle": { + "deliveredTokens": 7883, + "metrics": { + "recallAt1": 0.033333, + "recallAt5": 0.166667, + "recallAt10": 0.333333, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.380049, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13933", + "repositorySequence": 60, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4386, + "balancedV1": 4903, + "qarinahV2": 4386, + "qarinahV2NoGraph": 4386, + "oracle": 646 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 512, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 454, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 512, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 512, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 131, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.826235, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 885, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 980, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 885, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 885, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 646, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1948, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1978, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1948, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1948, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 646, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3877, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 3991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3877, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3877, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 646, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7954, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 7988, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7954, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7954, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 646, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.509836, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-13964", + "repositorySequence": 61, + "positiveUnderStructuralOracle": true, + "directRecords": 2, + "supportingRecords": 29, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.032258, + "recallAt5": 0.16129, + "recallAt10": 0.322581, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.659237, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.322581 + }, + "balancedV1": { + "recallAt1": 0.032258, + "recallAt5": 0.16129, + "recallAt10": 0.322581, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.582103, + "directRecallAt10": 0, + "supportingRecallAt10": 0.322581 + }, + "qarinahV2": { + "recallAt1": 0.032258, + "recallAt5": 0.16129, + "recallAt10": 0.322581, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.659237, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.322581 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.032258, + "recallAt5": 0.16129, + "recallAt10": 0.322581, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.659237, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.322581 + }, + "oracle": { + "recallAt1": 0.032258, + "recallAt5": 0.16129, + "recallAt10": 0.322581, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.322581 + } + }, + "volumes": { + "bm25Admitted": 5769, + "balancedV1": 4812, + "qarinahV2": 5769, + "qarinahV2NoGraph": 5769, + "oracle": 4847 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.032258, + "recallAt5": 0.032258, + "recallAt10": 0.032258, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.128116, + "directRecallAt10": 0, + "supportingRecallAt10": 0.032258 + } + }, + "balancedV1": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.032258, + "recallAt5": 0.032258, + "recallAt10": 0.032258, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.128116, + "directRecallAt10": 0, + "supportingRecallAt10": 0.032258 + } + }, + "qarinahV2": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.032258, + "recallAt5": 0.032258, + "recallAt10": 0.032258, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.128116, + "directRecallAt10": 0, + "supportingRecallAt10": 0.032258 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.032258, + "recallAt5": 0.032258, + "recallAt10": 0.032258, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.128116, + "directRecallAt10": 0, + "supportingRecallAt10": 0.032258 + } + }, + "oracle": { + "deliveredTokens": 408, + "metrics": { + "recallAt1": 0.032258, + "recallAt5": 0.064516, + "recallAt10": 0.064516, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.465181, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.064516 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 929, + "metrics": { + "recallAt1": 0.032258, + "recallAt5": 0.032258, + "recallAt10": 0.032258, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.128116, + "directRecallAt10": 0, + "supportingRecallAt10": 0.032258 + } + }, + "balancedV1": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.032258, + "recallAt5": 0.032258, + "recallAt10": 0.032258, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.128116, + "directRecallAt10": 0, + "supportingRecallAt10": 0.032258 + } + }, + "qarinahV2": { + "deliveredTokens": 929, + "metrics": { + "recallAt1": 0.032258, + "recallAt5": 0.032258, + "recallAt10": 0.032258, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.128116, + "directRecallAt10": 0, + "supportingRecallAt10": 0.032258 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 929, + "metrics": { + "recallAt1": 0.032258, + "recallAt5": 0.032258, + "recallAt10": 0.032258, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.128116, + "directRecallAt10": 0, + "supportingRecallAt10": 0.032258 + } + }, + "oracle": { + "deliveredTokens": 904, + "metrics": { + "recallAt1": 0.032258, + "recallAt5": 0.129032, + "recallAt10": 0.129032, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.584415, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.129032 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1927, + "metrics": { + "recallAt1": 0.032258, + "recallAt5": 0.064516, + "recallAt10": 0.064516, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.208948, + "directRecallAt10": 0, + "supportingRecallAt10": 0.064516 + } + }, + "balancedV1": { + "deliveredTokens": 1990, + "metrics": { + "recallAt1": 0.032258, + "recallAt5": 0.129032, + "recallAt10": 0.129032, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.438536, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.129032 + } + }, + "qarinahV2": { + "deliveredTokens": 1927, + "metrics": { + "recallAt1": 0.032258, + "recallAt5": 0.064516, + "recallAt10": 0.064516, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.208948, + "directRecallAt10": 0, + "supportingRecallAt10": 0.064516 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1927, + "metrics": { + "recallAt1": 0.032258, + "recallAt5": 0.064516, + "recallAt10": 0.064516, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.208948, + "directRecallAt10": 0, + "supportingRecallAt10": 0.064516 + } + }, + "oracle": { + "deliveredTokens": 1982, + "metrics": { + "recallAt1": 0.032258, + "recallAt5": 0.16129, + "recallAt10": 0.16129, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.795642, + "directRecallAt10": 1, + "supportingRecallAt10": 0.16129 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3917, + "metrics": { + "recallAt1": 0.032258, + "recallAt5": 0.16129, + "recallAt10": 0.258065, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.506503, + "directRecallAt10": 0, + "supportingRecallAt10": 0.258065 + } + }, + "balancedV1": { + "deliveredTokens": 3922, + "metrics": { + "recallAt1": 0.032258, + "recallAt5": 0.16129, + "recallAt10": 0.290323, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.545069, + "directRecallAt10": 0, + "supportingRecallAt10": 0.290323 + } + }, + "qarinahV2": { + "deliveredTokens": 3917, + "metrics": { + "recallAt1": 0.032258, + "recallAt5": 0.16129, + "recallAt10": 0.258065, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.506503, + "directRecallAt10": 0, + "supportingRecallAt10": 0.258065 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3917, + "metrics": { + "recallAt1": 0.032258, + "recallAt5": 0.16129, + "recallAt10": 0.258065, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.506503, + "directRecallAt10": 0, + "supportingRecallAt10": 0.258065 + } + }, + "oracle": { + "deliveredTokens": 3980, + "metrics": { + "recallAt1": 0.032258, + "recallAt5": 0.16129, + "recallAt10": 0.322581, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.322581 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 8000, + "metrics": { + "recallAt1": 0.032258, + "recallAt5": 0.16129, + "recallAt10": 0.322581, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.659237, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.322581 + } + }, + "balancedV1": { + "deliveredTokens": 7978, + "metrics": { + "recallAt1": 0.032258, + "recallAt5": 0.16129, + "recallAt10": 0.322581, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.582103, + "directRecallAt10": 0, + "supportingRecallAt10": 0.322581 + } + }, + "qarinahV2": { + "deliveredTokens": 8000, + "metrics": { + "recallAt1": 0.032258, + "recallAt5": 0.16129, + "recallAt10": 0.322581, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.659237, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.322581 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 8000, + "metrics": { + "recallAt1": 0.032258, + "recallAt5": 0.16129, + "recallAt10": 0.322581, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.659237, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.322581 + } + }, + "oracle": { + "deliveredTokens": 7918, + "metrics": { + "recallAt1": 0.032258, + "recallAt5": 0.16129, + "recallAt10": 0.322581, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.322581 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.511429, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-14016", + "repositorySequence": 62, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 32, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.03125, + "recallAt5": 0.0625, + "recallAt10": 0.15625, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.518119, + "directRecallAt10": null, + "supportingRecallAt10": 0.15625 + }, + "balancedV1": { + "recallAt1": 0.03125, + "recallAt5": 0.15625, + "recallAt10": 0.28125, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.933746, + "directRecallAt10": null, + "supportingRecallAt10": 0.28125 + }, + "qarinahV2": { + "recallAt1": 0.03125, + "recallAt5": 0.0625, + "recallAt10": 0.15625, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.518119, + "directRecallAt10": null, + "supportingRecallAt10": 0.15625 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.03125, + "recallAt5": 0.0625, + "recallAt10": 0.15625, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.518119, + "directRecallAt10": null, + "supportingRecallAt10": 0.15625 + }, + "oracle": { + "recallAt1": 0.03125, + "recallAt5": 0.15625, + "recallAt10": 0.3125, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.3125 + } + }, + "volumes": { + "bm25Admitted": 4989, + "balancedV1": 6076, + "qarinahV2": 4989, + "qarinahV2NoGraph": 4989, + "oracle": 4847 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 483, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.03125, + "recallAt10": 0.03125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.03125 + } + }, + "balancedV1": { + "deliveredTokens": 503, + "metrics": { + "recallAt1": 0.03125, + "recallAt5": 0.0625, + "recallAt10": 0.0625, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.0625 + } + }, + "qarinahV2": { + "deliveredTokens": 483, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.03125, + "recallAt10": 0.03125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.03125 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 483, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.03125, + "recallAt10": 0.03125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.03125 + } + }, + "oracle": { + "deliveredTokens": 408, + "metrics": { + "recallAt1": 0.03125, + "recallAt5": 0.0625, + "recallAt10": 0.0625, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.0625 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.03125, + "recallAt5": 0.03125, + "recallAt10": 0.03125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.03125 + } + }, + "balancedV1": { + "deliveredTokens": 905, + "metrics": { + "recallAt1": 0.03125, + "recallAt5": 0.0625, + "recallAt10": 0.0625, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.0625 + } + }, + "qarinahV2": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.03125, + "recallAt5": 0.03125, + "recallAt10": 0.03125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.03125 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.03125, + "recallAt5": 0.03125, + "recallAt10": 0.03125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.03125 + } + }, + "oracle": { + "deliveredTokens": 904, + "metrics": { + "recallAt1": 0.03125, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1982, + "metrics": { + "recallAt1": 0.03125, + "recallAt5": 0.0625, + "recallAt10": 0.0625, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.31488, + "directRecallAt10": null, + "supportingRecallAt10": 0.0625 + } + }, + "balancedV1": { + "deliveredTokens": 1983, + "metrics": { + "recallAt1": 0.03125, + "recallAt5": 0.09375, + "recallAt10": 0.09375, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.09375 + } + }, + "qarinahV2": { + "deliveredTokens": 1982, + "metrics": { + "recallAt1": 0.03125, + "recallAt5": 0.0625, + "recallAt10": 0.0625, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.31488, + "directRecallAt10": null, + "supportingRecallAt10": 0.0625 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1982, + "metrics": { + "recallAt1": 0.03125, + "recallAt5": 0.0625, + "recallAt10": 0.0625, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.31488, + "directRecallAt10": null, + "supportingRecallAt10": 0.0625 + } + }, + "oracle": { + "deliveredTokens": 1947, + "metrics": { + "recallAt1": 0.03125, + "recallAt5": 0.15625, + "recallAt10": 0.1875, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.1875 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3882, + "metrics": { + "recallAt1": 0.03125, + "recallAt5": 0.0625, + "recallAt10": 0.09375, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.388244, + "directRecallAt10": null, + "supportingRecallAt10": 0.09375 + } + }, + "balancedV1": { + "deliveredTokens": 3996, + "metrics": { + "recallAt1": 0.03125, + "recallAt5": 0.15625, + "recallAt10": 0.21875, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.800694, + "directRecallAt10": null, + "supportingRecallAt10": 0.21875 + } + }, + "qarinahV2": { + "deliveredTokens": 3882, + "metrics": { + "recallAt1": 0.03125, + "recallAt5": 0.0625, + "recallAt10": 0.09375, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.388244, + "directRecallAt10": null, + "supportingRecallAt10": 0.09375 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3882, + "metrics": { + "recallAt1": 0.03125, + "recallAt5": 0.0625, + "recallAt10": 0.09375, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.388244, + "directRecallAt10": null, + "supportingRecallAt10": 0.09375 + } + }, + "oracle": { + "deliveredTokens": 3980, + "metrics": { + "recallAt1": 0.03125, + "recallAt5": 0.15625, + "recallAt10": 0.3125, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.3125 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7926, + "metrics": { + "recallAt1": 0.03125, + "recallAt5": 0.0625, + "recallAt10": 0.15625, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.518119, + "directRecallAt10": null, + "supportingRecallAt10": 0.15625 + } + }, + "balancedV1": { + "deliveredTokens": 7987, + "metrics": { + "recallAt1": 0.03125, + "recallAt5": 0.15625, + "recallAt10": 0.28125, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.933746, + "directRecallAt10": null, + "supportingRecallAt10": 0.28125 + } + }, + "qarinahV2": { + "deliveredTokens": 7926, + "metrics": { + "recallAt1": 0.03125, + "recallAt5": 0.0625, + "recallAt10": 0.15625, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.518119, + "directRecallAt10": null, + "supportingRecallAt10": 0.15625 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7926, + "metrics": { + "recallAt1": 0.03125, + "recallAt5": 0.0625, + "recallAt10": 0.15625, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.518119, + "directRecallAt10": null, + "supportingRecallAt10": 0.15625 + } + }, + "oracle": { + "deliveredTokens": 7918, + "metrics": { + "recallAt1": 0.03125, + "recallAt5": 0.15625, + "recallAt10": 0.3125, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.3125 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "DIRECTLY_SUPPORTED", + "decision": "ACCEPT_DIRECT", + "score": 0.696939, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "CONSERVATIVE_DIRECT_THRESHOLD_MET" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-14017", + "repositorySequence": 63, + "positiveUnderStructuralOracle": true, + "directRecords": 3, + "supportingRecords": 31, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.088235, + "recallAt10": 0.147059, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.387666, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.147059 + }, + "balancedV1": { + "recallAt1": 0.029412, + "recallAt5": 0.147059, + "recallAt10": 0.264706, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.547465, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.264706 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.088235, + "recallAt10": 0.147059, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.387666, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.147059 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.088235, + "recallAt10": 0.147059, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.387666, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.147059 + }, + "oracle": { + "recallAt1": 0.029412, + "recallAt5": 0.147059, + "recallAt10": 0.294118, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.294118 + } + }, + "volumes": { + "bm25Admitted": 6817, + "balancedV1": 6411, + "qarinahV2": 6817, + "qarinahV2NoGraph": 6817, + "oracle": 4147 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 507, + "metrics": { + "recallAt1": 0.029412, + "recallAt5": 0.029412, + "recallAt10": 0.029412, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.340699, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.029412 + } + }, + "balancedV1": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.029412, + "recallAt5": 0.029412, + "recallAt10": 0.029412, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.113566, + "directRecallAt10": 0, + "supportingRecallAt10": 0.029412 + } + }, + "qarinahV2": { + "deliveredTokens": 507, + "metrics": { + "recallAt1": 0.029412, + "recallAt5": 0.029412, + "recallAt10": 0.029412, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.340699, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.029412 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 507, + "metrics": { + "recallAt1": 0.029412, + "recallAt5": 0.029412, + "recallAt10": 0.029412, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.340699, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.029412 + } + }, + "oracle": { + "deliveredTokens": 508, + "metrics": { + "recallAt1": 0.029412, + "recallAt5": 0.058824, + "recallAt10": 0.058824, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.555657, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.058824 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 989, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.058824, + "recallAt10": 0.058824, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.271741, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.058824 + } + }, + "balancedV1": { + "deliveredTokens": 992, + "metrics": { + "recallAt1": 0.029412, + "recallAt5": 0.058824, + "recallAt10": 0.058824, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.328524, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.058824 + } + }, + "qarinahV2": { + "deliveredTokens": 989, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.058824, + "recallAt10": 0.058824, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.271741, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.058824 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 989, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.058824, + "recallAt10": 0.058824, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.271741, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.058824 + } + }, + "oracle": { + "deliveredTokens": 963, + "metrics": { + "recallAt1": 0.029412, + "recallAt5": 0.088235, + "recallAt10": 0.088235, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.61244, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.088235 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1928, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.058824, + "recallAt10": 0.058824, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.271741, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.058824 + } + }, + "balancedV1": { + "deliveredTokens": 1947, + "metrics": { + "recallAt1": 0.029412, + "recallAt5": 0.088235, + "recallAt10": 0.088235, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.242002, + "directRecallAt10": 0, + "supportingRecallAt10": 0.088235 + } + }, + "qarinahV2": { + "deliveredTokens": 1928, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.058824, + "recallAt10": 0.058824, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.271741, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.058824 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1928, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.058824, + "recallAt10": 0.058824, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.271741, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.058824 + } + }, + "oracle": { + "deliveredTokens": 1895, + "metrics": { + "recallAt1": 0.029412, + "recallAt5": 0.147059, + "recallAt10": 0.176471, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.859303, + "directRecallAt10": 1, + "supportingRecallAt10": 0.176471 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3883, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.088235, + "recallAt10": 0.117647, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.358506, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.117647 + } + }, + "balancedV1": { + "deliveredTokens": 3959, + "metrics": { + "recallAt1": 0.029412, + "recallAt5": 0.147059, + "recallAt10": 0.176471, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.456206, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.176471 + } + }, + "qarinahV2": { + "deliveredTokens": 3883, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.088235, + "recallAt10": 0.117647, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.358506, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.117647 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3883, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.088235, + "recallAt10": 0.117647, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.358506, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.117647 + } + }, + "oracle": { + "deliveredTokens": 3893, + "metrics": { + "recallAt1": 0.029412, + "recallAt5": 0.147059, + "recallAt10": 0.294118, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.294118 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7890, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.088235, + "recallAt10": 0.147059, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.387666, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.147059 + } + }, + "balancedV1": { + "deliveredTokens": 7995, + "metrics": { + "recallAt1": 0.029412, + "recallAt5": 0.147059, + "recallAt10": 0.264706, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.547465, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.264706 + } + }, + "qarinahV2": { + "deliveredTokens": 7890, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.088235, + "recallAt10": 0.147059, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.387666, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.147059 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7890, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.088235, + "recallAt10": 0.147059, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.387666, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.147059 + } + }, + "oracle": { + "deliveredTokens": 7878, + "metrics": { + "recallAt1": 0.029412, + "recallAt5": 0.147059, + "recallAt10": 0.294118, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.294118 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.575, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-14155", + "repositorySequence": 64, + "positiveUnderStructuralOracle": true, + "directRecords": 2, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4070, + "balancedV1": 4571, + "qarinahV2": 4070, + "qarinahV2NoGraph": 4070, + "oracle": 811 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 503, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 503, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 503, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 296, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 920, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 989, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 920, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 920, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 811, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1987, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1987, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1987, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1987, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 811, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3974, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 3945, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3974, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3974, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 811, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7965, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 7960, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7965, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7965, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 811, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.41, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-14238", + "repositorySequence": 65, + "positiveUnderStructuralOracle": true, + "directRecords": 6, + "supportingRecords": 28, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.058824, + "recallAt10": 0.147059, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.164642, + "directRecallAt10": 0, + "supportingRecallAt10": 0.147059 + }, + "balancedV1": { + "recallAt1": 0.029412, + "recallAt5": 0.117647, + "recallAt10": 0.235294, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.323826, + "directRecallAt10": 0, + "supportingRecallAt10": 0.235294 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.058824, + "recallAt10": 0.147059, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.164642, + "directRecallAt10": 0, + "supportingRecallAt10": 0.147059 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.058824, + "recallAt10": 0.147059, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.164642, + "directRecallAt10": 0, + "supportingRecallAt10": 0.147059 + }, + "oracle": { + "recallAt1": 0.029412, + "recallAt5": 0.147059, + "recallAt10": 0.294118, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.294118 + } + }, + "volumes": { + "bm25Admitted": 8626, + "balancedV1": 6691, + "qarinahV2": 8626, + "qarinahV2NoGraph": 8626, + "oracle": 2826 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.029412, + "recallAt5": 0.029412, + "recallAt10": 0.029412, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.089663, + "directRecallAt10": 0, + "supportingRecallAt10": 0.029412 + } + }, + "qarinahV2": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.029412, + "recallAt5": 0.058824, + "recallAt10": 0.058824, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.438701, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.058824 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 936, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 969, + "metrics": { + "recallAt1": 0.029412, + "recallAt5": 0.058824, + "recallAt10": 0.058824, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.146234, + "directRecallAt10": 0, + "supportingRecallAt10": 0.058824 + } + }, + "qarinahV2": { + "deliveredTokens": 936, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 936, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 995, + "metrics": { + "recallAt1": 0.029412, + "recallAt5": 0.117647, + "recallAt10": 0.117647, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.611811, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.117647 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1874, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1897, + "metrics": { + "recallAt1": 0.029412, + "recallAt5": 0.058824, + "recallAt10": 0.058824, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.146234, + "directRecallAt10": 0, + "supportingRecallAt10": 0.058824 + } + }, + "qarinahV2": { + "deliveredTokens": 1874, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1874, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 1974, + "metrics": { + "recallAt1": 0.029412, + "recallAt5": 0.147059, + "recallAt10": 0.235294, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.94709, + "directRecallAt10": 1, + "supportingRecallAt10": 0.235294 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3902, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.029412, + "recallAt10": 0.029412, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.044831, + "directRecallAt10": 0, + "supportingRecallAt10": 0.029412 + } + }, + "balancedV1": { + "deliveredTokens": 3898, + "metrics": { + "recallAt1": 0.029412, + "recallAt5": 0.058824, + "recallAt10": 0.058824, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.134494, + "directRecallAt10": 0, + "supportingRecallAt10": 0.058824 + } + }, + "qarinahV2": { + "deliveredTokens": 3902, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.029412, + "recallAt10": 0.029412, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.044831, + "directRecallAt10": 0, + "supportingRecallAt10": 0.029412 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3902, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.029412, + "recallAt10": 0.029412, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.044831, + "directRecallAt10": 0, + "supportingRecallAt10": 0.029412 + } + }, + "oracle": { + "deliveredTokens": 3951, + "metrics": { + "recallAt1": 0.029412, + "recallAt5": 0.147059, + "recallAt10": 0.294118, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.294118 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7983, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.058824, + "recallAt10": 0.117647, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.138724, + "directRecallAt10": 0, + "supportingRecallAt10": 0.117647 + } + }, + "balancedV1": { + "deliveredTokens": 7964, + "metrics": { + "recallAt1": 0.029412, + "recallAt5": 0.117647, + "recallAt10": 0.235294, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.323826, + "directRecallAt10": 0, + "supportingRecallAt10": 0.235294 + } + }, + "qarinahV2": { + "deliveredTokens": 7983, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.058824, + "recallAt10": 0.117647, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.138724, + "directRecallAt10": 0, + "supportingRecallAt10": 0.117647 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7983, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.058824, + "recallAt10": 0.117647, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.138724, + "directRecallAt10": 0, + "supportingRecallAt10": 0.117647 + } + }, + "oracle": { + "deliveredTokens": 7940, + "metrics": { + "recallAt1": 0.029412, + "recallAt5": 0.147059, + "recallAt10": 0.294118, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.294118 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.462944, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-14382", + "repositorySequence": 66, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 5, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.292775, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.292775, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.292775, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + }, + "oracle": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 5324, + "balancedV1": 5372, + "qarinahV2": 5324, + "qarinahV2NoGraph": 5324, + "oracle": 2011 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 471, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 471, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 471, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 508, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.684478, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 903, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.213195, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 975, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 903, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.213195, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 903, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.213195, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 920, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.778735, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1980, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.175445, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 1991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.072927, + "directRecallAt10": 0, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2": { + "deliveredTokens": 1980, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.175445, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1980, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.175445, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 1793, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 0.833333, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.93285, + "directRecallAt10": 1, + "supportingRecallAt10": 0.833333 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3967, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.295031, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 3938, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3967, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.295031, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3967, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.295031, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 2011, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7898, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.292775, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 7996, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7898, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.292775, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7898, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.292775, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 2011, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.614214, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-14411", + "repositorySequence": 67, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 9, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.208451, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.208451, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.208451, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + }, + "oracle": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3758, + "balancedV1": 4193, + "qarinahV2": 3758, + "qarinahV2NoGraph": 3758, + "oracle": 3052 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 463, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 389, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 463, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 463, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 393, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.383343, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 929, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.218751, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + }, + "balancedV1": { + "deliveredTokens": 929, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 929, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.218751, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 929, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.218751, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + }, + "oracle": { + "deliveredTokens": 933, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.444444, + "recallAt10": 0.444444, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.602094, + "directRecallAt10": null, + "supportingRecallAt10": 0.444444 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1957, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.208451, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + }, + "balancedV1": { + "deliveredTokens": 1920, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1957, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.208451, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1957, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.208451, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + }, + "oracle": { + "deliveredTokens": 1955, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 0.555556, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.693022, + "directRecallAt10": null, + "supportingRecallAt10": 0.555556 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3909, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.208451, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + }, + "balancedV1": { + "deliveredTokens": 3914, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3909, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.208451, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3909, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.208451, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + }, + "oracle": { + "deliveredTokens": 3052, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7945, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.208451, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + }, + "balancedV1": { + "deliveredTokens": 7950, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7945, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.208451, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7945, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.208451, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + }, + "oracle": { + "deliveredTokens": 3052, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.55, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-14534", + "repositorySequence": 68, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.234639, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + }, + "balancedV1": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.604931, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.234639, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.234639, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3993, + "balancedV1": 4127, + "qarinahV2": 3993, + "qarinahV2NoGraph": 3993, + "oracle": 1010 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 425, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 495, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 425, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 425, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 495, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 954, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.703918, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 495, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1977, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.234639, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 1881, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636439, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 1977, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.234639, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1977, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.234639, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 1010, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3993, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.234639, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 3996, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 3993, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.234639, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3993, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.234639, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 1010, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7943, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.234639, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 7886, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.604931, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 7943, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.234639, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7943, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.234639, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 1010, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.38, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 8, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-14580", + "repositorySequence": 69, + "positiveUnderStructuralOracle": true, + "directRecords": 2, + "supportingRecords": 33, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.028571, + "recallAt5": 0.114286, + "recallAt10": 0.228571, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.836368, + "directRecallAt10": 1, + "supportingRecallAt10": 0.228571 + }, + "balancedV1": { + "recallAt1": 0.028571, + "recallAt5": 0.142857, + "recallAt10": 0.285714, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.285714 + }, + "qarinahV2": { + "recallAt1": 0.028571, + "recallAt5": 0.114286, + "recallAt10": 0.228571, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.836368, + "directRecallAt10": 1, + "supportingRecallAt10": 0.228571 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.028571, + "recallAt5": 0.114286, + "recallAt10": 0.228571, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.836368, + "directRecallAt10": 1, + "supportingRecallAt10": 0.228571 + }, + "oracle": { + "recallAt1": 0.028571, + "recallAt5": 0.142857, + "recallAt10": 0.285714, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.285714 + } + }, + "volumes": { + "bm25Admitted": 5000, + "balancedV1": 5311, + "qarinahV2": 5000, + "qarinahV2NoGraph": 5000, + "oracle": 4848 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 455, + "metrics": { + "recallAt1": 0.028571, + "recallAt5": 0.028571, + "recallAt10": 0.028571, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.384348, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.028571 + } + }, + "balancedV1": { + "deliveredTokens": 455, + "metrics": { + "recallAt1": 0.028571, + "recallAt5": 0.028571, + "recallAt10": 0.028571, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.384348, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.028571 + } + }, + "qarinahV2": { + "deliveredTokens": 455, + "metrics": { + "recallAt1": 0.028571, + "recallAt5": 0.028571, + "recallAt10": 0.028571, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.384348, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.028571 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 455, + "metrics": { + "recallAt1": 0.028571, + "recallAt5": 0.028571, + "recallAt10": 0.028571, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.384348, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.028571 + } + }, + "oracle": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.028571, + "recallAt5": 0.028571, + "recallAt10": 0.028571, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.384348, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.028571 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 951, + "metrics": { + "recallAt1": 0.028571, + "recallAt5": 0.085714, + "recallAt10": 0.085714, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.529239, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.085714 + } + }, + "balancedV1": { + "deliveredTokens": 914, + "metrics": { + "recallAt1": 0.028571, + "recallAt5": 0.057143, + "recallAt10": 0.057143, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.626845, + "directRecallAt10": 1, + "supportingRecallAt10": 0.057143 + } + }, + "qarinahV2": { + "deliveredTokens": 951, + "metrics": { + "recallAt1": 0.028571, + "recallAt5": 0.085714, + "recallAt10": 0.085714, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.529239, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.085714 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 951, + "metrics": { + "recallAt1": 0.028571, + "recallAt5": 0.085714, + "recallAt10": 0.085714, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.529239, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.085714 + } + }, + "oracle": { + "deliveredTokens": 914, + "metrics": { + "recallAt1": 0.028571, + "recallAt5": 0.057143, + "recallAt10": 0.057143, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.626845, + "directRecallAt10": 1, + "supportingRecallAt10": 0.057143 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1919, + "metrics": { + "recallAt1": 0.028571, + "recallAt5": 0.085714, + "recallAt10": 0.085714, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.63071, + "directRecallAt10": 1, + "supportingRecallAt10": 0.085714 + } + }, + "balancedV1": { + "deliveredTokens": 1925, + "metrics": { + "recallAt1": 0.028571, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.795642, + "directRecallAt10": 1, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2": { + "deliveredTokens": 1919, + "metrics": { + "recallAt1": 0.028571, + "recallAt5": 0.085714, + "recallAt10": 0.085714, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.63071, + "directRecallAt10": 1, + "supportingRecallAt10": 0.085714 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1919, + "metrics": { + "recallAt1": 0.028571, + "recallAt5": 0.085714, + "recallAt10": 0.085714, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.63071, + "directRecallAt10": 1, + "supportingRecallAt10": 0.085714 + } + }, + "oracle": { + "deliveredTokens": 1886, + "metrics": { + "recallAt1": 0.028571, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.795642, + "directRecallAt10": 1, + "supportingRecallAt10": 0.142857 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3909, + "metrics": { + "recallAt1": 0.028571, + "recallAt5": 0.114286, + "recallAt10": 0.2, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.797801, + "directRecallAt10": 1, + "supportingRecallAt10": 0.2 + } + }, + "balancedV1": { + "deliveredTokens": 3900, + "metrics": { + "recallAt1": 0.028571, + "recallAt5": 0.142857, + "recallAt10": 0.2, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.883983, + "directRecallAt10": 1, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2": { + "deliveredTokens": 3909, + "metrics": { + "recallAt1": 0.028571, + "recallAt5": 0.114286, + "recallAt10": 0.2, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.797801, + "directRecallAt10": 1, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3909, + "metrics": { + "recallAt1": 0.028571, + "recallAt5": 0.114286, + "recallAt10": 0.2, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.797801, + "directRecallAt10": 1, + "supportingRecallAt10": 0.2 + } + }, + "oracle": { + "deliveredTokens": 3981, + "metrics": { + "recallAt1": 0.028571, + "recallAt5": 0.142857, + "recallAt10": 0.285714, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.285714 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7901, + "metrics": { + "recallAt1": 0.028571, + "recallAt5": 0.114286, + "recallAt10": 0.228571, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.836368, + "directRecallAt10": 1, + "supportingRecallAt10": 0.228571 + } + }, + "balancedV1": { + "deliveredTokens": 7965, + "metrics": { + "recallAt1": 0.028571, + "recallAt5": 0.142857, + "recallAt10": 0.285714, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.285714 + } + }, + "qarinahV2": { + "deliveredTokens": 7901, + "metrics": { + "recallAt1": 0.028571, + "recallAt5": 0.114286, + "recallAt10": 0.228571, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.836368, + "directRecallAt10": 1, + "supportingRecallAt10": 0.228571 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7901, + "metrics": { + "recallAt1": 0.028571, + "recallAt5": 0.114286, + "recallAt10": 0.228571, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.836368, + "directRecallAt10": 1, + "supportingRecallAt10": 0.228571 + } + }, + "oracle": { + "deliveredTokens": 7887, + "metrics": { + "recallAt1": 0.028571, + "recallAt5": 0.142857, + "recallAt10": 0.285714, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.285714 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.557788, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-14608", + "repositorySequence": 70, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 4, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.616595, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.616595, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.616595, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2986, + "balancedV1": 3854, + "qarinahV2": 2986, + "qarinahV2NoGraph": 2986, + "oracle": 1478 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 500, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 495, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 500, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 500, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 495, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 968, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.19519, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 979, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 968, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.19519, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 968, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.19519, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 963, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1967, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.493444, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "balancedV1": { + "deliveredTokens": 1911, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 1967, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.493444, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1967, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.493444, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "oracle": { + "deliveredTokens": 1478, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3905, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.616595, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3992, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 3905, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.616595, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3905, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.616595, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1478, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7962, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.616595, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 7928, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 7962, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.616595, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7962, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.616595, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1478, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.376923, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "NO_CODE_ENTITY_MATCH", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-14667", + "repositorySequence": 71, + "positiveUnderStructuralOracle": true, + "directRecords": 4, + "supportingRecords": 32, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.027778, + "recallAt5": 0.138889, + "recallAt10": 0.25, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.641617, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.25 + }, + "balancedV1": { + "recallAt1": 0.027778, + "recallAt5": 0.138889, + "recallAt10": 0.277778, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.686672, + "directRecallAt10": 0.75, + "supportingRecallAt10": 0.277778 + }, + "qarinahV2": { + "recallAt1": 0.027778, + "recallAt5": 0.138889, + "recallAt10": 0.25, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.641617, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.25 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.027778, + "recallAt5": 0.138889, + "recallAt10": 0.25, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.641617, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.25 + }, + "oracle": { + "recallAt1": 0.027778, + "recallAt5": 0.138889, + "recallAt10": 0.277778, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.277778 + } + }, + "volumes": { + "bm25Admitted": 4741, + "balancedV1": 4365, + "qarinahV2": 4741, + "qarinahV2NoGraph": 4741, + "oracle": 4034 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 484, + "metrics": { + "recallAt1": 0.027778, + "recallAt5": 0.055556, + "recallAt10": 0.055556, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.375609, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.055556 + } + }, + "balancedV1": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0.027778, + "recallAt5": 0.055556, + "recallAt10": 0.055556, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.168715, + "directRecallAt10": 0, + "supportingRecallAt10": 0.055556 + } + }, + "qarinahV2": { + "deliveredTokens": 484, + "metrics": { + "recallAt1": 0.027778, + "recallAt5": 0.055556, + "recallAt10": 0.055556, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.375609, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.055556 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 484, + "metrics": { + "recallAt1": 0.027778, + "recallAt5": 0.055556, + "recallAt10": 0.055556, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.375609, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.055556 + } + }, + "oracle": { + "deliveredTokens": 484, + "metrics": { + "recallAt1": 0.027778, + "recallAt5": 0.055556, + "recallAt10": 0.055556, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.375609, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.055556 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.027778, + "recallAt5": 0.027778, + "recallAt10": 0.027778, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.103447, + "directRecallAt10": 0, + "supportingRecallAt10": 0.027778 + } + }, + "balancedV1": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.027778, + "recallAt5": 0.027778, + "recallAt10": 0.027778, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.103447, + "directRecallAt10": 0, + "supportingRecallAt10": 0.027778 + } + }, + "qarinahV2": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.027778, + "recallAt5": 0.027778, + "recallAt10": 0.027778, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.103447, + "directRecallAt10": 0, + "supportingRecallAt10": 0.027778 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.027778, + "recallAt5": 0.027778, + "recallAt10": 0.027778, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.103447, + "directRecallAt10": 0, + "supportingRecallAt10": 0.027778 + } + }, + "oracle": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0.027778, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.602421, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.111111 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1988, + "metrics": { + "recallAt1": 0.027778, + "recallAt5": 0.138889, + "recallAt10": 0.138889, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.435545, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.138889 + } + }, + "balancedV1": { + "deliveredTokens": 1988, + "metrics": { + "recallAt1": 0.027778, + "recallAt5": 0.138889, + "recallAt10": 0.138889, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.394114, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.138889 + } + }, + "qarinahV2": { + "deliveredTokens": 1988, + "metrics": { + "recallAt1": 0.027778, + "recallAt5": 0.138889, + "recallAt10": 0.138889, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.435545, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.138889 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1988, + "metrics": { + "recallAt1": 0.027778, + "recallAt5": 0.138889, + "recallAt10": 0.138889, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.435545, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.138889 + } + }, + "oracle": { + "deliveredTokens": 1945, + "metrics": { + "recallAt1": 0.027778, + "recallAt5": 0.138889, + "recallAt10": 0.166667, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.782736, + "directRecallAt10": 0.75, + "supportingRecallAt10": 0.166667 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3944, + "metrics": { + "recallAt1": 0.027778, + "recallAt5": 0.138889, + "recallAt10": 0.222222, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.611714, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.222222 + } + }, + "balancedV1": { + "deliveredTokens": 3985, + "metrics": { + "recallAt1": 0.027778, + "recallAt5": 0.138889, + "recallAt10": 0.277778, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.624391, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.277778 + } + }, + "qarinahV2": { + "deliveredTokens": 3944, + "metrics": { + "recallAt1": 0.027778, + "recallAt5": 0.138889, + "recallAt10": 0.222222, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.611714, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.222222 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3944, + "metrics": { + "recallAt1": 0.027778, + "recallAt5": 0.138889, + "recallAt10": 0.222222, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.611714, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.222222 + } + }, + "oracle": { + "deliveredTokens": 3934, + "metrics": { + "recallAt1": 0.027778, + "recallAt5": 0.138889, + "recallAt10": 0.277778, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.277778 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7972, + "metrics": { + "recallAt1": 0.027778, + "recallAt5": 0.138889, + "recallAt10": 0.25, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.641617, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 7976, + "metrics": { + "recallAt1": 0.027778, + "recallAt5": 0.138889, + "recallAt10": 0.277778, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.686672, + "directRecallAt10": 0.75, + "supportingRecallAt10": 0.277778 + } + }, + "qarinahV2": { + "deliveredTokens": 7972, + "metrics": { + "recallAt1": 0.027778, + "recallAt5": 0.138889, + "recallAt10": 0.25, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.641617, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7972, + "metrics": { + "recallAt1": 0.027778, + "recallAt5": 0.138889, + "recallAt10": 0.25, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.641617, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 7990, + "metrics": { + "recallAt1": 0.027778, + "recallAt5": 0.138889, + "recallAt10": 0.277778, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.277778 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.58, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-14672", + "repositorySequence": 72, + "positiveUnderStructuralOracle": true, + "directRecords": 3, + "supportingRecords": 35, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.026316, + "recallAt5": 0.078947, + "recallAt10": 0.184211, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.430691, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.184211 + }, + "balancedV1": { + "recallAt1": 0.026316, + "recallAt5": 0.131579, + "recallAt10": 0.263158, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.58437, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.263158 + }, + "qarinahV2": { + "recallAt1": 0.026316, + "recallAt5": 0.078947, + "recallAt10": 0.184211, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.430691, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.184211 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.026316, + "recallAt5": 0.078947, + "recallAt10": 0.184211, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.430691, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.184211 + }, + "oracle": { + "recallAt1": 0.026316, + "recallAt5": 0.131579, + "recallAt10": 0.263158, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.263158 + } + }, + "volumes": { + "bm25Admitted": 7807, + "balancedV1": 6265, + "qarinahV2": 7807, + "qarinahV2NoGraph": 7807, + "oracle": 4299 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 434, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.026316, + "recallAt10": 0.026316, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.071652, + "directRecallAt10": 0, + "supportingRecallAt10": 0.026316 + } + }, + "balancedV1": { + "deliveredTokens": 503, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.052632, + "recallAt10": 0.052632, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.185219, + "directRecallAt10": 0, + "supportingRecallAt10": 0.052632 + } + }, + "qarinahV2": { + "deliveredTokens": 434, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.026316, + "recallAt10": 0.026316, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.071652, + "directRecallAt10": 0, + "supportingRecallAt10": 0.026316 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 434, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.026316, + "recallAt10": 0.026316, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.071652, + "directRecallAt10": 0, + "supportingRecallAt10": 0.026316 + } + }, + "oracle": { + "deliveredTokens": 501, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.052632, + "recallAt10": 0.052632, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.412352, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.052632 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 949, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.026316, + "recallAt10": 0.026316, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.113566, + "directRecallAt10": 0, + "supportingRecallAt10": 0.026316 + } + }, + "balancedV1": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.052632, + "recallAt10": 0.052632, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.185219, + "directRecallAt10": 0, + "supportingRecallAt10": 0.052632 + } + }, + "qarinahV2": { + "deliveredTokens": 949, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.026316, + "recallAt10": 0.026316, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.113566, + "directRecallAt10": 0, + "supportingRecallAt10": 0.026316 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 949, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.026316, + "recallAt10": 0.026316, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.113566, + "directRecallAt10": 0, + "supportingRecallAt10": 0.026316 + } + }, + "oracle": { + "deliveredTokens": 984, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.052632, + "recallAt10": 0.052632, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.555657, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.052632 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1950, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.026316, + "recallAt10": 0.026316, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.113566, + "directRecallAt10": 0, + "supportingRecallAt10": 0.026316 + } + }, + "balancedV1": { + "deliveredTokens": 1879, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.078947, + "recallAt10": 0.078947, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.242002, + "directRecallAt10": 0, + "supportingRecallAt10": 0.078947 + } + }, + "qarinahV2": { + "deliveredTokens": 1950, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.026316, + "recallAt10": 0.026316, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.113566, + "directRecallAt10": 0, + "supportingRecallAt10": 0.026316 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1950, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.026316, + "recallAt10": 0.026316, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.113566, + "directRecallAt10": 0, + "supportingRecallAt10": 0.026316 + } + }, + "oracle": { + "deliveredTokens": 1876, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.131579, + "recallAt10": 0.131579, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.81885, + "directRecallAt10": 1, + "supportingRecallAt10": 0.131579 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3947, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.052632, + "recallAt10": 0.052632, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.17035, + "directRecallAt10": 0, + "supportingRecallAt10": 0.052632 + } + }, + "balancedV1": { + "deliveredTokens": 3979, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.131579, + "recallAt10": 0.157895, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.375299, + "directRecallAt10": 0, + "supportingRecallAt10": 0.157895 + } + }, + "qarinahV2": { + "deliveredTokens": 3947, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.052632, + "recallAt10": 0.052632, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.17035, + "directRecallAt10": 0, + "supportingRecallAt10": 0.052632 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3947, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.052632, + "recallAt10": 0.052632, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.17035, + "directRecallAt10": 0, + "supportingRecallAt10": 0.052632 + } + }, + "oracle": { + "deliveredTokens": 3984, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.131579, + "recallAt10": 0.263158, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.263158 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7945, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.078947, + "recallAt10": 0.184211, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.430691, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.184211 + } + }, + "balancedV1": { + "deliveredTokens": 7997, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.131579, + "recallAt10": 0.263158, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.58437, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.263158 + } + }, + "qarinahV2": { + "deliveredTokens": 7945, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.078947, + "recallAt10": 0.184211, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.430691, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.184211 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7945, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.078947, + "recallAt10": 0.184211, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.430691, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.184211 + } + }, + "oracle": { + "deliveredTokens": 7897, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.131579, + "recallAt10": 0.263158, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.263158 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.501633, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-14730", + "repositorySequence": 73, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 37, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.026316, + "recallAt5": 0.131579, + "recallAt10": 0.184211, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.861611, + "directRecallAt10": 1, + "supportingRecallAt10": 0.184211 + }, + "balancedV1": { + "recallAt1": 0.026316, + "recallAt5": 0.131579, + "recallAt10": 0.263158, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.694356, + "directRecallAt10": 0, + "supportingRecallAt10": 0.263158 + }, + "qarinahV2": { + "recallAt1": 0.026316, + "recallAt5": 0.131579, + "recallAt10": 0.184211, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.861611, + "directRecallAt10": 1, + "supportingRecallAt10": 0.184211 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.026316, + "recallAt5": 0.131579, + "recallAt10": 0.184211, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.861611, + "directRecallAt10": 1, + "supportingRecallAt10": 0.184211 + }, + "oracle": { + "recallAt1": 0.026316, + "recallAt5": 0.131579, + "recallAt10": 0.263158, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.263158 + } + }, + "volumes": { + "bm25Admitted": 3948, + "balancedV1": 4440, + "qarinahV2": 3948, + "qarinahV2NoGraph": 3948, + "oracle": 4647 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 411, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.052632, + "recallAt10": 0.052632, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.052632 + } + }, + "balancedV1": { + "deliveredTokens": 450, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.052632, + "recallAt10": 0.052632, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.052632 + } + }, + "qarinahV2": { + "deliveredTokens": 411, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.052632, + "recallAt10": 0.052632, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.052632 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 411, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.052632, + "recallAt10": 0.052632, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.052632 + } + }, + "oracle": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.052632, + "recallAt10": 0.052632, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.052632 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 993, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.078947, + "recallAt10": 0.078947, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.631297, + "directRecallAt10": 1, + "supportingRecallAt10": 0.078947 + } + }, + "balancedV1": { + "deliveredTokens": 990, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.078947, + "recallAt10": 0.078947, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.325653, + "directRecallAt10": 0, + "supportingRecallAt10": 0.078947 + } + }, + "qarinahV2": { + "deliveredTokens": 993, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.078947, + "recallAt10": 0.078947, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.631297, + "directRecallAt10": 1, + "supportingRecallAt10": 0.078947 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 993, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.078947, + "recallAt10": 0.078947, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.631297, + "directRecallAt10": 1, + "supportingRecallAt10": 0.078947 + } + }, + "oracle": { + "deliveredTokens": 992, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.105263, + "recallAt10": 0.105263, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.697114, + "directRecallAt10": 1, + "supportingRecallAt10": 0.105263 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1980, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.131579, + "recallAt10": 0.131579, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.756234, + "directRecallAt10": 1, + "supportingRecallAt10": 0.131579 + } + }, + "balancedV1": { + "deliveredTokens": 1897, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.078947, + "recallAt10": 0.078947, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.325653, + "directRecallAt10": 0, + "supportingRecallAt10": 0.078947 + } + }, + "qarinahV2": { + "deliveredTokens": 1980, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.131579, + "recallAt10": 0.131579, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.756234, + "directRecallAt10": 1, + "supportingRecallAt10": 0.131579 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1980, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.131579, + "recallAt10": 0.131579, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.756234, + "directRecallAt10": 1, + "supportingRecallAt10": 0.131579 + } + }, + "oracle": { + "deliveredTokens": 1987, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.131579, + "recallAt10": 0.184211, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.861611, + "directRecallAt10": 1, + "supportingRecallAt10": 0.184211 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3948, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.131579, + "recallAt10": 0.184211, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.861611, + "directRecallAt10": 1, + "supportingRecallAt10": 0.184211 + } + }, + "balancedV1": { + "deliveredTokens": 3985, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.131579, + "recallAt10": 0.236842, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.65018, + "directRecallAt10": 0, + "supportingRecallAt10": 0.236842 + } + }, + "qarinahV2": { + "deliveredTokens": 3948, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.131579, + "recallAt10": 0.184211, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.861611, + "directRecallAt10": 1, + "supportingRecallAt10": 0.184211 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3948, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.131579, + "recallAt10": 0.184211, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.861611, + "directRecallAt10": 1, + "supportingRecallAt10": 0.184211 + } + }, + "oracle": { + "deliveredTokens": 3928, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.131579, + "recallAt10": 0.263158, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.263158 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7935, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.131579, + "recallAt10": 0.184211, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.861611, + "directRecallAt10": 1, + "supportingRecallAt10": 0.184211 + } + }, + "balancedV1": { + "deliveredTokens": 7966, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.131579, + "recallAt10": 0.263158, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.694356, + "directRecallAt10": 0, + "supportingRecallAt10": 0.263158 + } + }, + "qarinahV2": { + "deliveredTokens": 7935, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.131579, + "recallAt10": 0.184211, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.861611, + "directRecallAt10": 1, + "supportingRecallAt10": 0.184211 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7935, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.131579, + "recallAt10": 0.184211, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.861611, + "directRecallAt10": 1, + "supportingRecallAt10": 0.184211 + } + }, + "oracle": { + "deliveredTokens": 7992, + "metrics": { + "recallAt1": 0.026316, + "recallAt5": 0.131579, + "recallAt10": 0.263158, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.263158 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.5775, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-14752", + "repositorySequence": 74, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 10, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.1, + "recallAt10": 0.3, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.2, + "ndcgAt10": 0.227938, + "directRecallAt10": null, + "supportingRecallAt10": 0.3 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.1, + "recallAt10": 0.3, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.2, + "ndcgAt10": 0.227938, + "directRecallAt10": null, + "supportingRecallAt10": 0.3 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.1, + "recallAt10": 0.3, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.2, + "ndcgAt10": 0.227938, + "directRecallAt10": null, + "supportingRecallAt10": 0.3 + }, + "oracle": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4921, + "balancedV1": 6316, + "qarinahV2": 4921, + "qarinahV2NoGraph": 4921, + "oracle": 3222 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 446, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.1 + } + }, + "balancedV1": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 446, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 446, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.1 + } + }, + "oracle": { + "deliveredTokens": 393, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 939, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.110046, + "directRecallAt10": null, + "supportingRecallAt10": 0.1 + } + }, + "balancedV1": { + "deliveredTokens": 914, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 939, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.110046, + "directRecallAt10": null, + "supportingRecallAt10": 0.1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 939, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.110046, + "directRecallAt10": null, + "supportingRecallAt10": 0.1 + } + }, + "oracle": { + "deliveredTokens": 965, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1969, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.1, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.163541, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "balancedV1": { + "deliveredTokens": 1991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1969, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.1, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.163541, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1969, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.1, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.163541, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "oracle": { + "deliveredTokens": 1974, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3968, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.1, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.158507, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "balancedV1": { + "deliveredTokens": 3917, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3968, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.1, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.158507, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3968, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.1, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.158507, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "oracle": { + "deliveredTokens": 3222, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7962, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.1, + "recallAt10": 0.3, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.2, + "ndcgAt10": 0.227938, + "directRecallAt10": null, + "supportingRecallAt10": 0.3 + } + }, + "balancedV1": { + "deliveredTokens": 7949, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7962, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.1, + "recallAt10": 0.3, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.2, + "ndcgAt10": 0.227938, + "directRecallAt10": null, + "supportingRecallAt10": 0.3 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7962, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.1, + "recallAt10": 0.3, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.2, + "ndcgAt10": 0.227938, + "directRecallAt10": null, + "supportingRecallAt10": 0.3 + } + }, + "oracle": { + "deliveredTokens": 3222, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.414769, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-14787", + "repositorySequence": 75, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 4, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "oracle": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 6955, + "balancedV1": 5167, + "qarinahV2": 6955, + "qarinahV2NoGraph": 6955, + "oracle": 1832 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 410, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 410, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 410, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 383, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 995, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 952, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 995, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 995, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 598, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1948, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1909, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1948, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1948, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 1832, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3951, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 3927, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3951, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3951, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 1832, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7893, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 7885, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7893, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7893, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 1832, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.391176, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "NO_CODE_ENTITY_MATCH", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-14855", + "repositorySequence": 76, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.400023, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.181818, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.144652, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.400023, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.400023, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 4781, + "balancedV1": 4683, + "qarinahV2": 4781, + "qarinahV2NoGraph": 4781, + "oracle": 3720 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 430, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 430, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 430, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 393, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 964, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 914, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 964, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 964, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 965, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1870, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.400023, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 1997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.085143, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2": { + "deliveredTokens": 1870, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.400023, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1870, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.400023, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 1974, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3872, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.400023, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 3909, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.181818, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.147829, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 3872, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.400023, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3872, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.400023, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 3720, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7936, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.400023, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 7995, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.181818, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.144652, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 7936, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.400023, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7936, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.400023, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 4010, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.541667, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-14915", + "repositorySequence": 77, + "positiveUnderStructuralOracle": true, + "directRecords": 7, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.1, + "recallAt5": 0.1, + "recallAt10": 0.4, + "precisionAt5": 0.2, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.314452, + "directRecallAt10": 0.428571, + "supportingRecallAt10": 0.4 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.1, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.126392, + "directRecallAt10": 0.142857, + "supportingRecallAt10": 0.2 + }, + "qarinahV2": { + "recallAt1": 0.1, + "recallAt5": 0.1, + "recallAt10": 0.4, + "precisionAt5": 0.2, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.314452, + "directRecallAt10": 0.428571, + "supportingRecallAt10": 0.4 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.1, + "recallAt5": 0.1, + "recallAt10": 0.4, + "precisionAt5": 0.2, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.314452, + "directRecallAt10": 0.428571, + "supportingRecallAt10": 0.4 + }, + "oracle": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 5814, + "balancedV1": 5549, + "qarinahV2": 5814, + "qarinahV2NoGraph": 5814, + "oracle": 4928 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 468, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.084606, + "directRecallAt10": 0, + "supportingRecallAt10": 0.1 + } + }, + "balancedV1": { + "deliveredTokens": 484, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.16014, + "directRecallAt10": 0.142857, + "supportingRecallAt10": 0.1 + } + }, + "qarinahV2": { + "deliveredTokens": 468, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.084606, + "directRecallAt10": 0, + "supportingRecallAt10": 0.1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 468, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.084606, + "directRecallAt10": 0, + "supportingRecallAt10": 0.1 + } + }, + "oracle": { + "deliveredTokens": 501, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.413957, + "directRecallAt10": 0.285714, + "supportingRecallAt10": 0.2 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 940, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.084606, + "directRecallAt10": 0, + "supportingRecallAt10": 0.1 + } + }, + "balancedV1": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.109313, + "directRecallAt10": 0.142857, + "supportingRecallAt10": 0.1 + } + }, + "qarinahV2": { + "deliveredTokens": 940, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.084606, + "directRecallAt10": 0, + "supportingRecallAt10": 0.1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 940, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.084606, + "directRecallAt10": 0, + "supportingRecallAt10": 0.1 + } + }, + "oracle": { + "deliveredTokens": 865, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.3, + "recallAt10": 0.3, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.540865, + "directRecallAt10": 0.428571, + "supportingRecallAt10": 0.3 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1999, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.182795, + "directRecallAt10": 0.142857, + "supportingRecallAt10": 0.2 + } + }, + "balancedV1": { + "deliveredTokens": 1989, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1999, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.182795, + "directRecallAt10": 0.142857, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1999, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.182795, + "directRecallAt10": 0.142857, + "supportingRecallAt10": 0.2 + } + }, + "oracle": { + "deliveredTokens": 1987, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 0.6, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.838779, + "directRecallAt10": 0.857143, + "supportingRecallAt10": 0.6 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3874, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.1, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.164676, + "directRecallAt10": 0.142857, + "supportingRecallAt10": 0.2 + } + }, + "balancedV1": { + "deliveredTokens": 3931, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.09819, + "directRecallAt10": 0.142857, + "supportingRecallAt10": 0.1 + } + }, + "qarinahV2": { + "deliveredTokens": 3874, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.1, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.164676, + "directRecallAt10": 0.142857, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3874, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.1, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.164676, + "directRecallAt10": 0.142857, + "supportingRecallAt10": 0.2 + } + }, + "oracle": { + "deliveredTokens": 3945, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 0.8, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.950075, + "directRecallAt10": 1, + "supportingRecallAt10": 0.8 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7931, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.1, + "recallAt10": 0.4, + "precisionAt5": 0.2, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.314452, + "directRecallAt10": 0.428571, + "supportingRecallAt10": 0.4 + } + }, + "balancedV1": { + "deliveredTokens": 7998, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.1, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.126392, + "directRecallAt10": 0.142857, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2": { + "deliveredTokens": 7931, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.1, + "recallAt10": 0.4, + "precisionAt5": 0.2, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.314452, + "directRecallAt10": 0.428571, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7931, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.1, + "recallAt10": 0.4, + "precisionAt5": 0.2, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.314452, + "directRecallAt10": 0.428571, + "supportingRecallAt10": 0.4 + } + }, + "oracle": { + "deliveredTokens": 4928, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.485714, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-14997", + "repositorySequence": 78, + "positiveUnderStructuralOracle": true, + "directRecords": 5, + "supportingRecords": 36, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.02439, + "recallAt5": 0.097561, + "recallAt10": 0.195122, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.425534, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.195122 + }, + "balancedV1": { + "recallAt1": 0.02439, + "recallAt5": 0.121951, + "recallAt10": 0.243902, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.490561, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.243902 + }, + "qarinahV2": { + "recallAt1": 0.02439, + "recallAt5": 0.097561, + "recallAt10": 0.195122, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.425534, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.195122 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.02439, + "recallAt5": 0.097561, + "recallAt10": 0.195122, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.425534, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.195122 + }, + "oracle": { + "recallAt1": 0.02439, + "recallAt5": 0.121951, + "recallAt10": 0.243902, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.243902 + } + }, + "volumes": { + "bm25Admitted": 8761, + "balancedV1": 6577, + "qarinahV2": 8761, + "qarinahV2NoGraph": 8761, + "oracle": 4917 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.04878, + "recallAt10": 0.04878, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.156212, + "directRecallAt10": 0, + "supportingRecallAt10": 0.04878 + } + }, + "balancedV1": { + "deliveredTokens": 455, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.02439, + "recallAt10": 0.02439, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.095781, + "directRecallAt10": 0, + "supportingRecallAt10": 0.02439 + } + }, + "qarinahV2": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.04878, + "recallAt10": 0.04878, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.156212, + "directRecallAt10": 0, + "supportingRecallAt10": 0.04878 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.04878, + "recallAt10": 0.04878, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.156212, + "directRecallAt10": 0, + "supportingRecallAt10": 0.04878 + } + }, + "oracle": { + "deliveredTokens": 501, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.04878, + "recallAt10": 0.04878, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.347774, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.04878 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 992, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.04878, + "recallAt10": 0.04878, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.156212, + "directRecallAt10": 0, + "supportingRecallAt10": 0.04878 + } + }, + "balancedV1": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.04878, + "recallAt10": 0.04878, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.156212, + "directRecallAt10": 0, + "supportingRecallAt10": 0.04878 + } + }, + "qarinahV2": { + "deliveredTokens": 992, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.04878, + "recallAt10": 0.04878, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.156212, + "directRecallAt10": 0, + "supportingRecallAt10": 0.04878 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 992, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.04878, + "recallAt10": 0.04878, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.156212, + "directRecallAt10": 0, + "supportingRecallAt10": 0.04878 + } + }, + "oracle": { + "deliveredTokens": 984, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.04878, + "recallAt10": 0.04878, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.468637, + "directRecallAt10": 0.4, + "supportingRecallAt10": 0.04878 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1994, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.04878, + "recallAt10": 0.04878, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.156212, + "directRecallAt10": 0, + "supportingRecallAt10": 0.04878 + } + }, + "balancedV1": { + "deliveredTokens": 1963, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.04878, + "recallAt10": 0.04878, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.156212, + "directRecallAt10": 0, + "supportingRecallAt10": 0.04878 + } + }, + "qarinahV2": { + "deliveredTokens": 1994, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.04878, + "recallAt10": 0.04878, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.156212, + "directRecallAt10": 0, + "supportingRecallAt10": 0.04878 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1994, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.04878, + "recallAt10": 0.04878, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.156212, + "directRecallAt10": 0, + "supportingRecallAt10": 0.04878 + } + }, + "oracle": { + "deliveredTokens": 1988, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.121951, + "recallAt10": 0.146341, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.807231, + "directRecallAt10": 0.8, + "supportingRecallAt10": 0.146341 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3968, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.121951, + "recallAt10": 0.121951, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.282407, + "directRecallAt10": 0, + "supportingRecallAt10": 0.121951 + } + }, + "balancedV1": { + "deliveredTokens": 3952, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.121951, + "recallAt10": 0.146341, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.316524, + "directRecallAt10": 0, + "supportingRecallAt10": 0.146341 + } + }, + "qarinahV2": { + "deliveredTokens": 3968, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.121951, + "recallAt10": 0.121951, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.282407, + "directRecallAt10": 0, + "supportingRecallAt10": 0.121951 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3968, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.121951, + "recallAt10": 0.121951, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.282407, + "directRecallAt10": 0, + "supportingRecallAt10": 0.121951 + } + }, + "oracle": { + "deliveredTokens": 3934, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.121951, + "recallAt10": 0.170732, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.913265, + "directRecallAt10": 1, + "supportingRecallAt10": 0.170732 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7958, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.097561, + "recallAt10": 0.170732, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.397847, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.170732 + } + }, + "balancedV1": { + "deliveredTokens": 7963, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.121951, + "recallAt10": 0.243902, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.490561, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.243902 + } + }, + "qarinahV2": { + "deliveredTokens": 7958, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.097561, + "recallAt10": 0.170732, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.397847, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.170732 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7958, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.097561, + "recallAt10": 0.170732, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.397847, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.170732 + } + }, + "oracle": { + "deliveredTokens": 7900, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.121951, + "recallAt10": 0.243902, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.243902 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.541371, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-14999", + "repositorySequence": 79, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 40, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.025, + "recallAt5": 0.1, + "recallAt10": 0.2, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.838957, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + }, + "balancedV1": { + "recallAt1": 0.025, + "recallAt5": 0.125, + "recallAt10": 0.25, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + }, + "qarinahV2": { + "recallAt1": 0.025, + "recallAt5": 0.1, + "recallAt10": 0.2, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.838957, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.025, + "recallAt5": 0.1, + "recallAt10": 0.2, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.838957, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + }, + "oracle": { + "recallAt1": 0.025, + "recallAt5": 0.125, + "recallAt10": 0.25, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "volumes": { + "bm25Admitted": 6703, + "balancedV1": 4822, + "qarinahV2": 6703, + "qarinahV2NoGraph": 6703, + "oracle": 4938 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 505, + "metrics": { + "recallAt1": 0.025, + "recallAt5": 0.025, + "recallAt10": 0.025, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.025 + } + }, + "balancedV1": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.025, + "recallAt5": 0.05, + "recallAt10": 0.05, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.05 + } + }, + "qarinahV2": { + "deliveredTokens": 505, + "metrics": { + "recallAt1": 0.025, + "recallAt5": 0.025, + "recallAt10": 0.025, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.025 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 505, + "metrics": { + "recallAt1": 0.025, + "recallAt5": 0.025, + "recallAt10": 0.025, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.025 + } + }, + "oracle": { + "deliveredTokens": 408, + "metrics": { + "recallAt1": 0.025, + "recallAt5": 0.05, + "recallAt10": 0.05, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.05 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 894, + "metrics": { + "recallAt1": 0.025, + "recallAt5": 0.05, + "recallAt10": 0.05, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.05 + } + }, + "balancedV1": { + "deliveredTokens": 902, + "metrics": { + "recallAt1": 0.025, + "recallAt5": 0.05, + "recallAt10": 0.05, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.05 + } + }, + "qarinahV2": { + "deliveredTokens": 894, + "metrics": { + "recallAt1": 0.025, + "recallAt5": 0.05, + "recallAt10": 0.05, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.05 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 894, + "metrics": { + "recallAt1": 0.025, + "recallAt5": 0.05, + "recallAt10": 0.05, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.05 + } + }, + "oracle": { + "deliveredTokens": 972, + "metrics": { + "recallAt1": 0.025, + "recallAt5": 0.075, + "recallAt10": 0.075, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.075 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1981, + "metrics": { + "recallAt1": 0.025, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.538886, + "directRecallAt10": null, + "supportingRecallAt10": 0.1 + } + }, + "balancedV1": { + "deliveredTokens": 1980, + "metrics": { + "recallAt1": 0.025, + "recallAt5": 0.075, + "recallAt10": 0.075, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.075 + } + }, + "qarinahV2": { + "deliveredTokens": 1981, + "metrics": { + "recallAt1": 0.025, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.538886, + "directRecallAt10": null, + "supportingRecallAt10": 0.1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1981, + "metrics": { + "recallAt1": 0.025, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.538886, + "directRecallAt10": null, + "supportingRecallAt10": 0.1 + } + }, + "oracle": { + "deliveredTokens": 1955, + "metrics": { + "recallAt1": 0.025, + "recallAt5": 0.125, + "recallAt10": 0.15, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.15 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3936, + "metrics": { + "recallAt1": 0.025, + "recallAt5": 0.1, + "recallAt10": 0.125, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.632541, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + }, + "balancedV1": { + "deliveredTokens": 3935, + "metrics": { + "recallAt1": 0.025, + "recallAt5": 0.125, + "recallAt10": 0.2, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.870125, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2": { + "deliveredTokens": 3936, + "metrics": { + "recallAt1": 0.025, + "recallAt5": 0.1, + "recallAt10": 0.125, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.632541, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3936, + "metrics": { + "recallAt1": 0.025, + "recallAt5": 0.1, + "recallAt10": 0.125, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.632541, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + }, + "oracle": { + "deliveredTokens": 3940, + "metrics": { + "recallAt1": 0.025, + "recallAt5": 0.125, + "recallAt10": 0.225, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.936379, + "directRecallAt10": null, + "supportingRecallAt10": 0.225 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7917, + "metrics": { + "recallAt1": 0.025, + "recallAt5": 0.1, + "recallAt10": 0.2, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.838957, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "balancedV1": { + "deliveredTokens": 7982, + "metrics": { + "recallAt1": 0.025, + "recallAt5": 0.125, + "recallAt10": 0.25, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2": { + "deliveredTokens": 7917, + "metrics": { + "recallAt1": 0.025, + "recallAt5": 0.1, + "recallAt10": 0.2, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.838957, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7917, + "metrics": { + "recallAt1": 0.025, + "recallAt5": 0.1, + "recallAt10": 0.2, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.838957, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "oracle": { + "deliveredTokens": 7916, + "metrics": { + "recallAt1": 0.025, + "recallAt5": 0.125, + "recallAt10": 0.25, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.555263, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15061", + "repositorySequence": 80, + "positiveUnderStructuralOracle": true, + "directRecords": 2, + "supportingRecords": 5, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.428571, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.481672, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.428571 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.285714, + "recallAt10": 0.428571, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.273463, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.428571 + }, + "qarinahV2": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.428571, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.481672, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.428571 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.428571, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.481672, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.428571 + }, + "oracle": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4145, + "balancedV1": 3623, + "qarinahV2": 4145, + "qarinahV2NoGraph": 4145, + "oracle": 2160 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 468, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.14493, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + } + }, + "balancedV1": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 468, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.14493, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 468, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.14493, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + } + }, + "oracle": { + "deliveredTokens": 507, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.428571, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.598698, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.428571 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.428571, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.491719, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.428571 + } + }, + "balancedV1": { + "deliveredTokens": 971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.072465, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.428571, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.491719, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.428571 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.428571, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.491719, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.428571 + } + }, + "oracle": { + "deliveredTokens": 816, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.428571, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.78158, + "directRecallAt10": 1, + "supportingRecallAt10": 0.428571 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1997, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.428571, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.481672, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.428571 + } + }, + "balancedV1": { + "deliveredTokens": 1974, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.285714, + "recallAt10": 0.428571, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.283408, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.428571 + } + }, + "qarinahV2": { + "deliveredTokens": 1997, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.428571, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.481672, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.428571 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1997, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.428571, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.481672, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.428571 + } + }, + "oracle": { + "deliveredTokens": 1692, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 0.857143, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.95169, + "directRecallAt10": 1, + "supportingRecallAt10": 0.857143 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3992, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.571429, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.523566, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.571429 + } + }, + "balancedV1": { + "deliveredTokens": 3929, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.285714, + "recallAt10": 0.428571, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.273463, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.428571 + } + }, + "qarinahV2": { + "deliveredTokens": 3992, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.571429, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.523566, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.571429 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3992, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.571429, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.523566, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.571429 + } + }, + "oracle": { + "deliveredTokens": 2160, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7985, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.428571, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.481672, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.428571 + } + }, + "balancedV1": { + "deliveredTokens": 7984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.285714, + "recallAt10": 0.428571, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.273463, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.428571 + } + }, + "qarinahV2": { + "deliveredTokens": 7985, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.428571, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.481672, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.428571 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7985, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.428571, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.481672, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.428571 + } + }, + "oracle": { + "deliveredTokens": 2160, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.57069, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15202", + "repositorySequence": 81, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 7, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.142857, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.082746, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.142857, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.082746, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.142857, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.082746, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + }, + "oracle": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 9369, + "balancedV1": 4589, + "qarinahV2": 9369, + "qarinahV2NoGraph": 9369, + "oracle": 2238 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 495, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 495, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 495, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 375, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.274876, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 949, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 946, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 949, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 949, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 863, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.428571, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.585742, + "directRecallAt10": null, + "supportingRecallAt10": 0.428571 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1952, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 1793, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 0.714286, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.810462, + "directRecallAt10": null, + "supportingRecallAt10": 0.714286 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3983, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.142857, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.097913, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "balancedV1": { + "deliveredTokens": 3913, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3983, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.142857, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.097913, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3983, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.142857, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.097913, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "oracle": { + "deliveredTokens": 2238, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7933, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.142857, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.082746, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "balancedV1": { + "deliveredTokens": 7984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7933, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.142857, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.082746, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7933, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.142857, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.082746, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "oracle": { + "deliveredTokens": 2238, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.493357, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15213", + "repositorySequence": 82, + "positiveUnderStructuralOracle": true, + "directRecords": 7, + "supportingRecords": 34, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.02439, + "recallAt5": 0.121951, + "recallAt10": 0.195122, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.332251, + "directRecallAt10": 0, + "supportingRecallAt10": 0.195122 + }, + "balancedV1": { + "recallAt1": 0.02439, + "recallAt5": 0.097561, + "recallAt10": 0.195122, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.306574, + "directRecallAt10": 0, + "supportingRecallAt10": 0.195122 + }, + "qarinahV2": { + "recallAt1": 0.02439, + "recallAt5": 0.121951, + "recallAt10": 0.195122, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.332251, + "directRecallAt10": 0, + "supportingRecallAt10": 0.195122 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.02439, + "recallAt5": 0.121951, + "recallAt10": 0.195122, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.332251, + "directRecallAt10": 0, + "supportingRecallAt10": 0.195122 + }, + "oracle": { + "recallAt1": 0.02439, + "recallAt5": 0.121951, + "recallAt10": 0.243902, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.243902 + } + }, + "volumes": { + "bm25Admitted": 4908, + "balancedV1": 4243, + "qarinahV2": 4908, + "qarinahV2NoGraph": 4908, + "oracle": 3189 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 423, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.02439, + "recallAt10": 0.02439, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.084606, + "directRecallAt10": 0, + "supportingRecallAt10": 0.02439 + } + }, + "balancedV1": { + "deliveredTokens": 495, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 423, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.02439, + "recallAt10": 0.02439, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.084606, + "directRecallAt10": 0, + "supportingRecallAt10": 0.02439 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 423, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.02439, + "recallAt10": 0.02439, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.084606, + "directRecallAt10": 0, + "supportingRecallAt10": 0.02439 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.04878, + "recallAt10": 0.04878, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.413957, + "directRecallAt10": 0.285714, + "supportingRecallAt10": 0.04878 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 911, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.04878, + "recallAt10": 0.04878, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.137986, + "directRecallAt10": 0, + "supportingRecallAt10": 0.04878 + } + }, + "balancedV1": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.02439, + "recallAt10": 0.02439, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.084606, + "directRecallAt10": 0, + "supportingRecallAt10": 0.02439 + } + }, + "qarinahV2": { + "deliveredTokens": 911, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.04878, + "recallAt10": 0.04878, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.137986, + "directRecallAt10": 0, + "supportingRecallAt10": 0.04878 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 911, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.04878, + "recallAt10": 0.04878, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.137986, + "directRecallAt10": 0, + "supportingRecallAt10": 0.04878 + } + }, + "oracle": { + "deliveredTokens": 988, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.097561, + "recallAt10": 0.097561, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.577303, + "directRecallAt10": 0.428571, + "supportingRecallAt10": 0.097561 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1909, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.073171, + "recallAt10": 0.073171, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.180288, + "directRecallAt10": 0, + "supportingRecallAt10": 0.073171 + } + }, + "balancedV1": { + "deliveredTokens": 1962, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.073171, + "recallAt10": 0.073171, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.163346, + "directRecallAt10": 0, + "supportingRecallAt10": 0.073171 + } + }, + "qarinahV2": { + "deliveredTokens": 1909, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.073171, + "recallAt10": 0.073171, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.180288, + "directRecallAt10": 0, + "supportingRecallAt10": 0.073171 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1909, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.073171, + "recallAt10": 0.073171, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.180288, + "directRecallAt10": 0, + "supportingRecallAt10": 0.073171 + } + }, + "oracle": { + "deliveredTokens": 1905, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.121951, + "recallAt10": 0.146341, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.838779, + "directRecallAt10": 0.857143, + "supportingRecallAt10": 0.146341 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3982, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.121951, + "recallAt10": 0.195122, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.332251, + "directRecallAt10": 0, + "supportingRecallAt10": 0.195122 + } + }, + "balancedV1": { + "deliveredTokens": 3937, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.097561, + "recallAt10": 0.195122, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.306574, + "directRecallAt10": 0, + "supportingRecallAt10": 0.195122 + } + }, + "qarinahV2": { + "deliveredTokens": 3982, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.121951, + "recallAt10": 0.195122, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.332251, + "directRecallAt10": 0, + "supportingRecallAt10": 0.195122 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3982, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.121951, + "recallAt10": 0.195122, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.332251, + "directRecallAt10": 0, + "supportingRecallAt10": 0.195122 + } + }, + "oracle": { + "deliveredTokens": 3904, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.121951, + "recallAt10": 0.243902, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.243902 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7968, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.121951, + "recallAt10": 0.195122, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.332251, + "directRecallAt10": 0, + "supportingRecallAt10": 0.195122 + } + }, + "balancedV1": { + "deliveredTokens": 7928, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.097561, + "recallAt10": 0.195122, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.306574, + "directRecallAt10": 0, + "supportingRecallAt10": 0.195122 + } + }, + "qarinahV2": { + "deliveredTokens": 7968, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.121951, + "recallAt10": 0.195122, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.332251, + "directRecallAt10": 0, + "supportingRecallAt10": 0.195122 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7968, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.121951, + "recallAt10": 0.195122, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.332251, + "directRecallAt10": 0, + "supportingRecallAt10": 0.195122 + } + }, + "oracle": { + "deliveredTokens": 7877, + "metrics": { + "recallAt1": 0.02439, + "recallAt5": 0.121951, + "recallAt10": 0.243902, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.243902 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "DIRECTLY_SUPPORTED", + "decision": "ACCEPT_DIRECT", + "score": 0.688462, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "CONSERVATIVE_DIRECT_THRESHOLD_MET" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15252", + "repositorySequence": 83, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 42, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.02381, + "recallAt5": 0.119048, + "recallAt10": 0.190476, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.864315, + "directRecallAt10": null, + "supportingRecallAt10": 0.190476 + }, + "balancedV1": { + "recallAt1": 0.02381, + "recallAt5": 0.119048, + "recallAt10": 0.238095, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.238095 + }, + "qarinahV2": { + "recallAt1": 0.02381, + "recallAt5": 0.119048, + "recallAt10": 0.190476, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.864315, + "directRecallAt10": null, + "supportingRecallAt10": 0.190476 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.02381, + "recallAt5": 0.119048, + "recallAt10": 0.190476, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.864315, + "directRecallAt10": null, + "supportingRecallAt10": 0.190476 + }, + "oracle": { + "recallAt1": 0.02381, + "recallAt5": 0.119048, + "recallAt10": 0.238095, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.238095 + } + }, + "volumes": { + "bm25Admitted": 7597, + "balancedV1": 5342, + "qarinahV2": 7597, + "qarinahV2NoGraph": 7597, + "oracle": 4938 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 455, + "metrics": { + "recallAt1": 0.02381, + "recallAt5": 0.02381, + "recallAt10": 0.02381, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.02381 + } + }, + "balancedV1": { + "deliveredTokens": 455, + "metrics": { + "recallAt1": 0.02381, + "recallAt5": 0.02381, + "recallAt10": 0.02381, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.02381 + } + }, + "qarinahV2": { + "deliveredTokens": 455, + "metrics": { + "recallAt1": 0.02381, + "recallAt5": 0.02381, + "recallAt10": 0.02381, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.02381 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 455, + "metrics": { + "recallAt1": 0.02381, + "recallAt5": 0.02381, + "recallAt10": 0.02381, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.02381 + } + }, + "oracle": { + "deliveredTokens": 408, + "metrics": { + "recallAt1": 0.02381, + "recallAt5": 0.047619, + "recallAt10": 0.047619, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.047619 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 961, + "metrics": { + "recallAt1": 0.02381, + "recallAt5": 0.047619, + "recallAt10": 0.047619, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.330138, + "directRecallAt10": null, + "supportingRecallAt10": 0.047619 + } + }, + "balancedV1": { + "deliveredTokens": 914, + "metrics": { + "recallAt1": 0.02381, + "recallAt5": 0.047619, + "recallAt10": 0.047619, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.047619 + } + }, + "qarinahV2": { + "deliveredTokens": 961, + "metrics": { + "recallAt1": 0.02381, + "recallAt5": 0.047619, + "recallAt10": 0.047619, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.330138, + "directRecallAt10": null, + "supportingRecallAt10": 0.047619 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 961, + "metrics": { + "recallAt1": 0.02381, + "recallAt5": 0.047619, + "recallAt10": 0.047619, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.330138, + "directRecallAt10": null, + "supportingRecallAt10": 0.047619 + } + }, + "oracle": { + "deliveredTokens": 972, + "metrics": { + "recallAt1": 0.02381, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.071429 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1943, + "metrics": { + "recallAt1": 0.02381, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.071429 + } + }, + "balancedV1": { + "deliveredTokens": 1910, + "metrics": { + "recallAt1": 0.02381, + "recallAt5": 0.095238, + "recallAt10": 0.095238, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.095238 + } + }, + "qarinahV2": { + "deliveredTokens": 1943, + "metrics": { + "recallAt1": 0.02381, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.071429 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1943, + "metrics": { + "recallAt1": 0.02381, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.071429 + } + }, + "oracle": { + "deliveredTokens": 1955, + "metrics": { + "recallAt1": 0.02381, + "recallAt5": 0.119048, + "recallAt10": 0.142857, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3916, + "metrics": { + "recallAt1": 0.02381, + "recallAt5": 0.095238, + "recallAt10": 0.095238, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.095238 + } + }, + "balancedV1": { + "deliveredTokens": 3881, + "metrics": { + "recallAt1": 0.02381, + "recallAt5": 0.119048, + "recallAt10": 0.142857, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2": { + "deliveredTokens": 3916, + "metrics": { + "recallAt1": 0.02381, + "recallAt5": 0.095238, + "recallAt10": 0.095238, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.095238 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3916, + "metrics": { + "recallAt1": 0.02381, + "recallAt5": 0.095238, + "recallAt10": 0.095238, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.095238 + } + }, + "oracle": { + "deliveredTokens": 3940, + "metrics": { + "recallAt1": 0.02381, + "recallAt5": 0.119048, + "recallAt10": 0.214286, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.936379, + "directRecallAt10": null, + "supportingRecallAt10": 0.214286 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7962, + "metrics": { + "recallAt1": 0.02381, + "recallAt5": 0.119048, + "recallAt10": 0.190476, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.864315, + "directRecallAt10": null, + "supportingRecallAt10": 0.190476 + } + }, + "balancedV1": { + "deliveredTokens": 7963, + "metrics": { + "recallAt1": 0.02381, + "recallAt5": 0.119048, + "recallAt10": 0.238095, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.238095 + } + }, + "qarinahV2": { + "deliveredTokens": 7962, + "metrics": { + "recallAt1": 0.02381, + "recallAt5": 0.119048, + "recallAt10": 0.190476, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.864315, + "directRecallAt10": null, + "supportingRecallAt10": 0.190476 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7962, + "metrics": { + "recallAt1": 0.02381, + "recallAt5": 0.119048, + "recallAt10": 0.190476, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.864315, + "directRecallAt10": null, + "supportingRecallAt10": 0.190476 + } + }, + "oracle": { + "deliveredTokens": 7916, + "metrics": { + "recallAt1": 0.02381, + "recallAt5": 0.119048, + "recallAt10": 0.238095, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.238095 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.617376, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15320", + "repositorySequence": 84, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 43, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.023256, + "recallAt5": 0.116279, + "recallAt10": 0.186047, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.863015, + "directRecallAt10": null, + "supportingRecallAt10": 0.186047 + }, + "balancedV1": { + "recallAt1": 0.023256, + "recallAt5": 0.116279, + "recallAt10": 0.232558, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.232558 + }, + "qarinahV2": { + "recallAt1": 0.023256, + "recallAt5": 0.116279, + "recallAt10": 0.186047, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.863015, + "directRecallAt10": null, + "supportingRecallAt10": 0.186047 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.023256, + "recallAt5": 0.116279, + "recallAt10": 0.186047, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.863015, + "directRecallAt10": null, + "supportingRecallAt10": 0.186047 + }, + "oracle": { + "recallAt1": 0.023256, + "recallAt5": 0.116279, + "recallAt10": 0.232558, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.232558 + } + }, + "volumes": { + "bm25Admitted": 5287, + "balancedV1": 5277, + "qarinahV2": 5287, + "qarinahV2NoGraph": 5287, + "oracle": 4938 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 484, + "metrics": { + "recallAt1": 0.023256, + "recallAt5": 0.046512, + "recallAt10": 0.046512, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.046512 + } + }, + "balancedV1": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.023256, + "recallAt5": 0.023256, + "recallAt10": 0.023256, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.023256 + } + }, + "qarinahV2": { + "deliveredTokens": 484, + "metrics": { + "recallAt1": 0.023256, + "recallAt5": 0.046512, + "recallAt10": 0.046512, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.046512 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 484, + "metrics": { + "recallAt1": 0.023256, + "recallAt5": 0.046512, + "recallAt10": 0.046512, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.046512 + } + }, + "oracle": { + "deliveredTokens": 408, + "metrics": { + "recallAt1": 0.023256, + "recallAt5": 0.046512, + "recallAt10": 0.046512, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.046512 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.023256, + "recallAt5": 0.023256, + "recallAt10": 0.023256, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.023256 + } + }, + "balancedV1": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.023256, + "recallAt5": 0.023256, + "recallAt10": 0.023256, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.023256 + } + }, + "qarinahV2": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.023256, + "recallAt5": 0.023256, + "recallAt10": 0.023256, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.023256 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.023256, + "recallAt5": 0.023256, + "recallAt10": 0.023256, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.023256 + } + }, + "oracle": { + "deliveredTokens": 972, + "metrics": { + "recallAt1": 0.023256, + "recallAt5": 0.069767, + "recallAt10": 0.069767, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.069767 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1990, + "metrics": { + "recallAt1": 0.023256, + "recallAt5": 0.069767, + "recallAt10": 0.069767, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.069767 + } + }, + "balancedV1": { + "deliveredTokens": 1958, + "metrics": { + "recallAt1": 0.023256, + "recallAt5": 0.093023, + "recallAt10": 0.093023, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.093023 + } + }, + "qarinahV2": { + "deliveredTokens": 1990, + "metrics": { + "recallAt1": 0.023256, + "recallAt5": 0.069767, + "recallAt10": 0.069767, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.069767 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1990, + "metrics": { + "recallAt1": 0.023256, + "recallAt5": 0.069767, + "recallAt10": 0.069767, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.069767 + } + }, + "oracle": { + "deliveredTokens": 1955, + "metrics": { + "recallAt1": 0.023256, + "recallAt5": 0.116279, + "recallAt10": 0.139535, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.139535 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3952, + "metrics": { + "recallAt1": 0.023256, + "recallAt5": 0.116279, + "recallAt10": 0.162791, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.793584, + "directRecallAt10": null, + "supportingRecallAt10": 0.162791 + } + }, + "balancedV1": { + "deliveredTokens": 3998, + "metrics": { + "recallAt1": 0.023256, + "recallAt5": 0.116279, + "recallAt10": 0.186047, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.870125, + "directRecallAt10": null, + "supportingRecallAt10": 0.186047 + } + }, + "qarinahV2": { + "deliveredTokens": 3952, + "metrics": { + "recallAt1": 0.023256, + "recallAt5": 0.116279, + "recallAt10": 0.162791, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.793584, + "directRecallAt10": null, + "supportingRecallAt10": 0.162791 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3952, + "metrics": { + "recallAt1": 0.023256, + "recallAt5": 0.116279, + "recallAt10": 0.162791, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.793584, + "directRecallAt10": null, + "supportingRecallAt10": 0.162791 + } + }, + "oracle": { + "deliveredTokens": 3940, + "metrics": { + "recallAt1": 0.023256, + "recallAt5": 0.116279, + "recallAt10": 0.209302, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.936379, + "directRecallAt10": null, + "supportingRecallAt10": 0.209302 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7962, + "metrics": { + "recallAt1": 0.023256, + "recallAt5": 0.116279, + "recallAt10": 0.186047, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.863015, + "directRecallAt10": null, + "supportingRecallAt10": 0.186047 + } + }, + "balancedV1": { + "deliveredTokens": 7887, + "metrics": { + "recallAt1": 0.023256, + "recallAt5": 0.116279, + "recallAt10": 0.232558, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.232558 + } + }, + "qarinahV2": { + "deliveredTokens": 7962, + "metrics": { + "recallAt1": 0.023256, + "recallAt5": 0.116279, + "recallAt10": 0.186047, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.863015, + "directRecallAt10": null, + "supportingRecallAt10": 0.186047 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7962, + "metrics": { + "recallAt1": 0.023256, + "recallAt5": 0.116279, + "recallAt10": 0.186047, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.863015, + "directRecallAt10": null, + "supportingRecallAt10": 0.186047 + } + }, + "oracle": { + "deliveredTokens": 7916, + "metrics": { + "recallAt1": 0.023256, + "recallAt5": 0.116279, + "recallAt10": 0.232558, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.232558 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.558798, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15347", + "repositorySequence": 85, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 12, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.083333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.078398, + "directRecallAt10": null, + "supportingRecallAt10": 0.083333 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.083333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.078398, + "directRecallAt10": null, + "supportingRecallAt10": 0.083333 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.083333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.078398, + "directRecallAt10": null, + "supportingRecallAt10": 0.083333 + }, + "oracle": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.833333, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.833333 + } + }, + "volumes": { + "bm25Admitted": 4816, + "balancedV1": 4847, + "qarinahV2": 4816, + "qarinahV2NoGraph": 4816, + "oracle": 3893 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.083333, + "recallAt10": 0.083333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.083333 + } + }, + "balancedV1": { + "deliveredTokens": 512, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.083333, + "recallAt10": 0.083333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.083333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.083333, + "recallAt10": 0.083333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.083333 + } + }, + "oracle": { + "deliveredTokens": 492, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 995, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.083333, + "recallAt10": 0.083333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.094788, + "directRecallAt10": null, + "supportingRecallAt10": 0.083333 + } + }, + "balancedV1": { + "deliveredTokens": 965, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 995, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.083333, + "recallAt10": 0.083333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.094788, + "directRecallAt10": null, + "supportingRecallAt10": 0.083333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 995, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.083333, + "recallAt10": 0.083333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.094788, + "directRecallAt10": null, + "supportingRecallAt10": 0.083333 + } + }, + "oracle": { + "deliveredTokens": 879, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1992, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1987, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1992, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1992, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 1869, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3892, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.083333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.078398, + "directRecallAt10": null, + "supportingRecallAt10": 0.083333 + } + }, + "balancedV1": { + "deliveredTokens": 3986, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3892, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.083333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.078398, + "directRecallAt10": null, + "supportingRecallAt10": 0.083333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3892, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.083333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.078398, + "directRecallAt10": null, + "supportingRecallAt10": 0.083333 + } + }, + "oracle": { + "deliveredTokens": 3893, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.833333, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.833333 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7990, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.083333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.078398, + "directRecallAt10": null, + "supportingRecallAt10": 0.083333 + } + }, + "balancedV1": { + "deliveredTokens": 7953, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7990, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.083333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.078398, + "directRecallAt10": null, + "supportingRecallAt10": 0.083333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7990, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.083333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.078398, + "directRecallAt10": null, + "supportingRecallAt10": 0.083333 + } + }, + "oracle": { + "deliveredTokens": 4326, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.833333, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.833333 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.453473, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15388", + "repositorySequence": 86, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 5994, + "balancedV1": 5591, + "qarinahV2": 5994, + "qarinahV2NoGraph": 5994, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 456, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 456, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 456, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 456, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 951, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 983, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 951, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 951, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1950, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1978, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1950, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1950, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3978, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3975, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3978, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3978, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7972, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 7998, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 7972, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7972, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.64403, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15400", + "repositorySequence": 87, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 5, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.2, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.098039, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.4, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.215151, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.2, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.098039, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.2, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.098039, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + }, + "oracle": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4991, + "balancedV1": 5007, + "qarinahV2": 4991, + "qarinahV2NoGraph": 4991, + "oracle": 2073 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 501, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 501, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 501, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 501, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 395, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.553146, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 996, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 996, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 996, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 839, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.8, + "recallAt10": 0.8, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.868795, + "directRecallAt10": null, + "supportingRecallAt10": 0.8 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1964, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1982, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1964, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1964, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 1858, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.8, + "recallAt10": 0.8, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.868795, + "directRecallAt10": null, + "supportingRecallAt10": 0.8 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3945, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.2, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.113053, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "balancedV1": { + "deliveredTokens": 3991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.2, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.113053, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2": { + "deliveredTokens": 3945, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.2, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.113053, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3945, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.2, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.113053, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "oracle": { + "deliveredTokens": 2073, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7947, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.2, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.098039, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "balancedV1": { + "deliveredTokens": 7993, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.4, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.215151, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2": { + "deliveredTokens": 7947, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.2, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.098039, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7947, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.2, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.098039, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "oracle": { + "deliveredTokens": 2073, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.4075, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15498", + "repositorySequence": 88, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3691, + "balancedV1": 4244, + "qarinahV2": 3691, + "qarinahV2NoGraph": 3691, + "oracle": 772 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 421, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 450, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 421, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 421, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 295, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 962, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 990, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 962, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 962, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 772, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1983, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1881, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1983, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1983, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 772, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3959, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 3954, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3959, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3959, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 772, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7931, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 7892, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7931, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7931, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 772, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.523333, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15695", + "repositorySequence": 89, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 43, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.022727, + "recallAt5": 0.113636, + "recallAt10": 0.227273, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.812595, + "directRecallAt10": 1, + "supportingRecallAt10": 0.227273 + }, + "balancedV1": { + "recallAt1": 0.022727, + "recallAt5": 0.113636, + "recallAt10": 0.227273, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.694356, + "directRecallAt10": 0, + "supportingRecallAt10": 0.227273 + }, + "qarinahV2": { + "recallAt1": 0.022727, + "recallAt5": 0.113636, + "recallAt10": 0.227273, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.812595, + "directRecallAt10": 1, + "supportingRecallAt10": 0.227273 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.022727, + "recallAt5": 0.113636, + "recallAt10": 0.227273, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.812595, + "directRecallAt10": 1, + "supportingRecallAt10": 0.227273 + }, + "oracle": { + "recallAt1": 0.022727, + "recallAt5": 0.113636, + "recallAt10": 0.227273, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.227273 + } + }, + "volumes": { + "bm25Admitted": 7142, + "balancedV1": 6951, + "qarinahV2": 7142, + "qarinahV2NoGraph": 7142, + "oracle": 3806 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 486, + "metrics": { + "recallAt1": 0.022727, + "recallAt5": 0.045455, + "recallAt10": 0.045455, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.442082, + "directRecallAt10": 1, + "supportingRecallAt10": 0.045455 + } + }, + "balancedV1": { + "deliveredTokens": 486, + "metrics": { + "recallAt1": 0.022727, + "recallAt5": 0.045455, + "recallAt10": 0.045455, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.442082, + "directRecallAt10": 1, + "supportingRecallAt10": 0.045455 + } + }, + "qarinahV2": { + "deliveredTokens": 486, + "metrics": { + "recallAt1": 0.022727, + "recallAt5": 0.045455, + "recallAt10": 0.045455, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.442082, + "directRecallAt10": 1, + "supportingRecallAt10": 0.045455 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 486, + "metrics": { + "recallAt1": 0.022727, + "recallAt5": 0.045455, + "recallAt10": 0.045455, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.442082, + "directRecallAt10": 1, + "supportingRecallAt10": 0.045455 + } + }, + "oracle": { + "deliveredTokens": 492, + "metrics": { + "recallAt1": 0.022727, + "recallAt5": 0.068182, + "recallAt10": 0.068182, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.631297, + "directRecallAt10": 1, + "supportingRecallAt10": 0.068182 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 966, + "metrics": { + "recallAt1": 0.022727, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.544292, + "directRecallAt10": 1, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 922, + "metrics": { + "recallAt1": 0.022727, + "recallAt5": 0.045455, + "recallAt10": 0.045455, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.442082, + "directRecallAt10": 1, + "supportingRecallAt10": 0.045455 + } + }, + "qarinahV2": { + "deliveredTokens": 966, + "metrics": { + "recallAt1": 0.022727, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.544292, + "directRecallAt10": 1, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 966, + "metrics": { + "recallAt1": 0.022727, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.544292, + "directRecallAt10": 1, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 950, + "metrics": { + "recallAt1": 0.022727, + "recallAt5": 0.113636, + "recallAt10": 0.113636, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.756234, + "directRecallAt10": 1, + "supportingRecallAt10": 0.113636 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1963, + "metrics": { + "recallAt1": 0.022727, + "recallAt5": 0.113636, + "recallAt10": 0.113636, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.582223, + "directRecallAt10": 1, + "supportingRecallAt10": 0.113636 + } + }, + "balancedV1": { + "deliveredTokens": 1928, + "metrics": { + "recallAt1": 0.022727, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.39147, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2": { + "deliveredTokens": 1963, + "metrics": { + "recallAt1": 0.022727, + "recallAt5": 0.113636, + "recallAt10": 0.113636, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.582223, + "directRecallAt10": 1, + "supportingRecallAt10": 0.113636 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1963, + "metrics": { + "recallAt1": 0.022727, + "recallAt5": 0.113636, + "recallAt10": 0.113636, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.582223, + "directRecallAt10": 1, + "supportingRecallAt10": 0.113636 + } + }, + "oracle": { + "deliveredTokens": 1973, + "metrics": { + "recallAt1": 0.022727, + "recallAt5": 0.113636, + "recallAt10": 0.159091, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.861611, + "directRecallAt10": 1, + "supportingRecallAt10": 0.159091 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3983, + "metrics": { + "recallAt1": 0.022727, + "recallAt5": 0.113636, + "recallAt10": 0.159091, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.674206, + "directRecallAt10": 1, + "supportingRecallAt10": 0.159091 + } + }, + "balancedV1": { + "deliveredTokens": 3976, + "metrics": { + "recallAt1": 0.022727, + "recallAt5": 0.113636, + "recallAt10": 0.136364, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.505026, + "directRecallAt10": 0, + "supportingRecallAt10": 0.136364 + } + }, + "qarinahV2": { + "deliveredTokens": 3983, + "metrics": { + "recallAt1": 0.022727, + "recallAt5": 0.113636, + "recallAt10": 0.159091, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.674206, + "directRecallAt10": 1, + "supportingRecallAt10": 0.159091 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3983, + "metrics": { + "recallAt1": 0.022727, + "recallAt5": 0.113636, + "recallAt10": 0.159091, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.674206, + "directRecallAt10": 1, + "supportingRecallAt10": 0.159091 + } + }, + "oracle": { + "deliveredTokens": 3937, + "metrics": { + "recallAt1": 0.022727, + "recallAt5": 0.113636, + "recallAt10": 0.227273, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.227273 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7916, + "metrics": { + "recallAt1": 0.022727, + "recallAt5": 0.113636, + "recallAt10": 0.227273, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.812595, + "directRecallAt10": 1, + "supportingRecallAt10": 0.227273 + } + }, + "balancedV1": { + "deliveredTokens": 7967, + "metrics": { + "recallAt1": 0.022727, + "recallAt5": 0.113636, + "recallAt10": 0.227273, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.694356, + "directRecallAt10": 0, + "supportingRecallAt10": 0.227273 + } + }, + "qarinahV2": { + "deliveredTokens": 7916, + "metrics": { + "recallAt1": 0.022727, + "recallAt5": 0.113636, + "recallAt10": 0.227273, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.812595, + "directRecallAt10": 1, + "supportingRecallAt10": 0.227273 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7916, + "metrics": { + "recallAt1": 0.022727, + "recallAt5": 0.113636, + "recallAt10": 0.227273, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.812595, + "directRecallAt10": 1, + "supportingRecallAt10": 0.227273 + } + }, + "oracle": { + "deliveredTokens": 7988, + "metrics": { + "recallAt1": 0.022727, + "recallAt5": 0.113636, + "recallAt10": 0.227273, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.227273 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.505882, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15738", + "repositorySequence": 90, + "positiveUnderStructuralOracle": true, + "directRecords": 2, + "supportingRecords": 43, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.022222, + "recallAt5": 0.088889, + "recallAt10": 0.177778, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.553976, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.177778 + }, + "balancedV1": { + "recallAt1": 0.022222, + "recallAt5": 0.111111, + "recallAt10": 0.222222, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.766638, + "directRecallAt10": 1, + "supportingRecallAt10": 0.222222 + }, + "qarinahV2": { + "recallAt1": 0.022222, + "recallAt5": 0.088889, + "recallAt10": 0.177778, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.553976, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.177778 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.022222, + "recallAt5": 0.088889, + "recallAt10": 0.177778, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.553976, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.177778 + }, + "oracle": { + "recallAt1": 0.022222, + "recallAt5": 0.111111, + "recallAt10": 0.222222, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.222222 + } + }, + "volumes": { + "bm25Admitted": 5007, + "balancedV1": 5959, + "qarinahV2": 5007, + "qarinahV2NoGraph": 5007, + "oracle": 3529 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 440, + "metrics": { + "recallAt1": 0.022222, + "recallAt5": 0.044444, + "recallAt10": 0.044444, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.208948, + "directRecallAt10": 0, + "supportingRecallAt10": 0.044444 + } + }, + "balancedV1": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.022222, + "recallAt5": 0.022222, + "recallAt10": 0.022222, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.128116, + "directRecallAt10": 0, + "supportingRecallAt10": 0.022222 + } + }, + "qarinahV2": { + "deliveredTokens": 440, + "metrics": { + "recallAt1": 0.022222, + "recallAt5": 0.044444, + "recallAt10": 0.044444, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.208948, + "directRecallAt10": 0, + "supportingRecallAt10": 0.044444 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 440, + "metrics": { + "recallAt1": 0.022222, + "recallAt5": 0.044444, + "recallAt10": 0.044444, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.208948, + "directRecallAt10": 0, + "supportingRecallAt10": 0.044444 + } + }, + "oracle": { + "deliveredTokens": 486, + "metrics": { + "recallAt1": 0.022222, + "recallAt5": 0.044444, + "recallAt10": 0.044444, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.465181, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.044444 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 938, + "metrics": { + "recallAt1": 0.022222, + "recallAt5": 0.044444, + "recallAt10": 0.044444, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.192174, + "directRecallAt10": 0, + "supportingRecallAt10": 0.044444 + } + }, + "balancedV1": { + "deliveredTokens": 914, + "metrics": { + "recallAt1": 0.022222, + "recallAt5": 0.044444, + "recallAt10": 0.044444, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.208948, + "directRecallAt10": 0, + "supportingRecallAt10": 0.044444 + } + }, + "qarinahV2": { + "deliveredTokens": 938, + "metrics": { + "recallAt1": 0.022222, + "recallAt5": 0.044444, + "recallAt10": 0.044444, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.192174, + "directRecallAt10": 0, + "supportingRecallAt10": 0.044444 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 938, + "metrics": { + "recallAt1": 0.022222, + "recallAt5": 0.044444, + "recallAt10": 0.044444, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.192174, + "directRecallAt10": 0, + "supportingRecallAt10": 0.044444 + } + }, + "oracle": { + "deliveredTokens": 907, + "metrics": { + "recallAt1": 0.022222, + "recallAt5": 0.044444, + "recallAt10": 0.044444, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.626845, + "directRecallAt10": 1, + "supportingRecallAt10": 0.044444 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1933, + "metrics": { + "recallAt1": 0.022222, + "recallAt5": 0.088889, + "recallAt10": 0.088889, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.296913, + "directRecallAt10": 0, + "supportingRecallAt10": 0.088889 + } + }, + "balancedV1": { + "deliveredTokens": 1993, + "metrics": { + "recallAt1": 0.022222, + "recallAt5": 0.111111, + "recallAt10": 0.133333, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.423381, + "directRecallAt10": 0, + "supportingRecallAt10": 0.133333 + } + }, + "qarinahV2": { + "deliveredTokens": 1933, + "metrics": { + "recallAt1": 0.022222, + "recallAt5": 0.088889, + "recallAt10": 0.088889, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.296913, + "directRecallAt10": 0, + "supportingRecallAt10": 0.088889 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1933, + "metrics": { + "recallAt1": 0.022222, + "recallAt5": 0.088889, + "recallAt10": 0.088889, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.296913, + "directRecallAt10": 0, + "supportingRecallAt10": 0.088889 + } + }, + "oracle": { + "deliveredTokens": 1982, + "metrics": { + "recallAt1": 0.022222, + "recallAt5": 0.111111, + "recallAt10": 0.133333, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.841278, + "directRecallAt10": 1, + "supportingRecallAt10": 0.133333 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3987, + "metrics": { + "recallAt1": 0.022222, + "recallAt5": 0.088889, + "recallAt10": 0.155556, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.516942, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.155556 + } + }, + "balancedV1": { + "deliveredTokens": 3983, + "metrics": { + "recallAt1": 0.022222, + "recallAt5": 0.111111, + "recallAt10": 0.222222, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.766638, + "directRecallAt10": 1, + "supportingRecallAt10": 0.222222 + } + }, + "qarinahV2": { + "deliveredTokens": 3987, + "metrics": { + "recallAt1": 0.022222, + "recallAt5": 0.088889, + "recallAt10": 0.155556, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.516942, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.155556 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3987, + "metrics": { + "recallAt1": 0.022222, + "recallAt5": 0.088889, + "recallAt10": 0.155556, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.516942, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.155556 + } + }, + "oracle": { + "deliveredTokens": 3932, + "metrics": { + "recallAt1": 0.022222, + "recallAt5": 0.111111, + "recallAt10": 0.222222, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.222222 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7938, + "metrics": { + "recallAt1": 0.022222, + "recallAt5": 0.088889, + "recallAt10": 0.177778, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.553976, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.177778 + } + }, + "balancedV1": { + "deliveredTokens": 7887, + "metrics": { + "recallAt1": 0.022222, + "recallAt5": 0.111111, + "recallAt10": 0.222222, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.766638, + "directRecallAt10": 1, + "supportingRecallAt10": 0.222222 + } + }, + "qarinahV2": { + "deliveredTokens": 7938, + "metrics": { + "recallAt1": 0.022222, + "recallAt5": 0.088889, + "recallAt10": 0.177778, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.553976, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.177778 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7938, + "metrics": { + "recallAt1": 0.022222, + "recallAt5": 0.088889, + "recallAt10": 0.177778, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.553976, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.177778 + } + }, + "oracle": { + "deliveredTokens": 7935, + "metrics": { + "recallAt1": 0.022222, + "recallAt5": 0.111111, + "recallAt10": 0.222222, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.222222 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.502955, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15781", + "repositorySequence": 91, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 8, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.125, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.412532, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.125, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.076143, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + }, + "qarinahV2": { + "recallAt1": 0.125, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.412532, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.125, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.412532, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + }, + "oracle": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4596, + "balancedV1": 4685, + "qarinahV2": 4596, + "qarinahV2NoGraph": 4596, + "oracle": 2424 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.252943, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + }, + "balancedV1": { + "deliveredTokens": 508, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.252943, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.252943, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + }, + "oracle": { + "deliveredTokens": 375, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.252943, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 919, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.252943, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + }, + "balancedV1": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 919, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.252943, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 919, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.252943, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + }, + "oracle": { + "deliveredTokens": 816, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.375, + "recallAt10": 0.375, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.539003, + "directRecallAt10": null, + "supportingRecallAt10": 0.375 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1989, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.375, + "recallAt10": 0.375, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.510383, + "directRecallAt10": null, + "supportingRecallAt10": 0.375 + } + }, + "balancedV1": { + "deliveredTokens": 1995, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1989, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.375, + "recallAt10": 0.375, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.510383, + "directRecallAt10": null, + "supportingRecallAt10": 0.375 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1989, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.375, + "recallAt10": 0.375, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.510383, + "directRecallAt10": null, + "supportingRecallAt10": 0.375 + } + }, + "oracle": { + "deliveredTokens": 1979, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 0.75, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.835891, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3887, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.25, + "recallAt10": 0.375, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.488675, + "directRecallAt10": null, + "supportingRecallAt10": 0.375 + } + }, + "balancedV1": { + "deliveredTokens": 3982, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.125, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.076143, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + }, + "qarinahV2": { + "deliveredTokens": 3887, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.25, + "recallAt10": 0.375, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.488675, + "directRecallAt10": null, + "supportingRecallAt10": 0.375 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3887, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.25, + "recallAt10": 0.375, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.488675, + "directRecallAt10": null, + "supportingRecallAt10": 0.375 + } + }, + "oracle": { + "deliveredTokens": 2424, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7895, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.412532, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 7945, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.125, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.076143, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + }, + "qarinahV2": { + "deliveredTokens": 7895, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.412532, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7895, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.412532, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 2424, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.526923, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15789", + "repositorySequence": 92, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 6, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.166667, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.091092, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.166667, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.091092, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.166667, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.091092, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + }, + "oracle": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4895, + "balancedV1": 5504, + "qarinahV2": 4895, + "qarinahV2NoGraph": 4895, + "oracle": 2321 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 492, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 492, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 492, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 489, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.493523, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 914, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 872, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.775148, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1999, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1992, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1999, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1999, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 1877, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.775148, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3923, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 3990, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3923, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3923, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 2321, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7992, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.166667, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.091092, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "balancedV1": { + "deliveredTokens": 7997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7992, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.166667, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.091092, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7992, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.166667, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.091092, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "oracle": { + "deliveredTokens": 2321, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.553409, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15790", + "repositorySequence": 93, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 9, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.222222, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.216241, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.111111, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.078349, + "directRecallAt10": null, + "supportingRecallAt10": 0.111111 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.222222, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.216241, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.222222, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.216241, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + }, + "oracle": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3778, + "balancedV1": 3431, + "qarinahV2": 3778, + "qarinahV2NoGraph": 3778, + "oracle": 2794 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 501, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.148297, + "directRecallAt10": null, + "supportingRecallAt10": 0.111111 + } + }, + "balancedV1": { + "deliveredTokens": 450, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 501, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.148297, + "directRecallAt10": null, + "supportingRecallAt10": 0.111111 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 501, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.148297, + "directRecallAt10": null, + "supportingRecallAt10": 0.111111 + } + }, + "oracle": { + "deliveredTokens": 375, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.235046, + "directRecallAt10": null, + "supportingRecallAt10": 0.111111 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.148297, + "directRecallAt10": null, + "supportingRecallAt10": 0.111111 + } + }, + "balancedV1": { + "deliveredTokens": 941, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.148297, + "directRecallAt10": null, + "supportingRecallAt10": 0.111111 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.148297, + "directRecallAt10": null, + "supportingRecallAt10": 0.111111 + } + }, + "oracle": { + "deliveredTokens": 816, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.500866, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1896, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.148297, + "directRecallAt10": null, + "supportingRecallAt10": 0.111111 + } + }, + "balancedV1": { + "deliveredTokens": 1881, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1896, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.148297, + "directRecallAt10": null, + "supportingRecallAt10": 0.111111 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1896, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.148297, + "directRecallAt10": null, + "supportingRecallAt10": 0.111111 + } + }, + "oracle": { + "deliveredTokens": 1927, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 0.777778, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.855096, + "directRecallAt10": null, + "supportingRecallAt10": 0.777778 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.222222, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.216241, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + }, + "balancedV1": { + "deliveredTokens": 3951, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.111111, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.078349, + "directRecallAt10": null, + "supportingRecallAt10": 0.111111 + } + }, + "qarinahV2": { + "deliveredTokens": 3991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.222222, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.216241, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.222222, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.216241, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + }, + "oracle": { + "deliveredTokens": 2794, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7897, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.222222, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.216241, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + }, + "balancedV1": { + "deliveredTokens": 8000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.111111, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.078349, + "directRecallAt10": null, + "supportingRecallAt10": 0.111111 + } + }, + "qarinahV2": { + "deliveredTokens": 7897, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.222222, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.216241, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7897, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.222222, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.216241, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + }, + "oracle": { + "deliveredTokens": 2794, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.395161, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "NO_CODE_ENTITY_MATCH", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15814", + "repositorySequence": 94, + "positiveUnderStructuralOracle": true, + "directRecords": 5, + "supportingRecords": 41, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.065217, + "recallAt10": 0.152174, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 0.5, + "ndcgAt10": 0.343429, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.152174 + }, + "balancedV1": { + "recallAt1": 0.021739, + "recallAt5": 0.108696, + "recallAt10": 0.217391, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.805031, + "directRecallAt10": 0.6, + "supportingRecallAt10": 0.217391 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.065217, + "recallAt10": 0.152174, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 0.5, + "ndcgAt10": 0.343429, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.152174 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.065217, + "recallAt10": 0.152174, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 0.5, + "ndcgAt10": 0.343429, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.152174 + }, + "oracle": { + "recallAt1": 0.021739, + "recallAt5": 0.108696, + "recallAt10": 0.217391, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.217391 + } + }, + "volumes": { + "bm25Admitted": 9952, + "balancedV1": 5891, + "qarinahV2": 9952, + "qarinahV2NoGraph": 9952, + "oracle": 4032 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.021739, + "recallAt10": 0.021739, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.060431, + "directRecallAt10": 0, + "supportingRecallAt10": 0.021739 + } + }, + "balancedV1": { + "deliveredTokens": 484, + "metrics": { + "recallAt1": 0.021739, + "recallAt5": 0.021739, + "recallAt10": 0.021739, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.287343, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.021739 + } + }, + "qarinahV2": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.021739, + "recallAt10": 0.021739, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.060431, + "directRecallAt10": 0, + "supportingRecallAt10": 0.021739 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.021739, + "recallAt10": 0.021739, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.060431, + "directRecallAt10": 0, + "supportingRecallAt10": 0.021739 + } + }, + "oracle": { + "deliveredTokens": 477, + "metrics": { + "recallAt1": 0.021739, + "recallAt5": 0.043478, + "recallAt10": 0.043478, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.347774, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.043478 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 993, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.021739, + "recallAt10": 0.021739, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.181293, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.021739 + } + }, + "balancedV1": { + "deliveredTokens": 991, + "metrics": { + "recallAt1": 0.021739, + "recallAt5": 0.043478, + "recallAt10": 0.043478, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.347774, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.043478 + } + }, + "qarinahV2": { + "deliveredTokens": 993, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.021739, + "recallAt10": 0.021739, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.181293, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.021739 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 993, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.021739, + "recallAt10": 0.021739, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.181293, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.021739 + } + }, + "oracle": { + "deliveredTokens": 955, + "metrics": { + "recallAt1": 0.021739, + "recallAt5": 0.065217, + "recallAt10": 0.065217, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.612308, + "directRecallAt10": 0.6, + "supportingRecallAt10": 0.065217 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.021739, + "recallAt10": 0.021739, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.060431, + "directRecallAt10": 0, + "supportingRecallAt10": 0.021739 + } + }, + "balancedV1": { + "deliveredTokens": 1989, + "metrics": { + "recallAt1": 0.021739, + "recallAt5": 0.065217, + "recallAt10": 0.065217, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.395665, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.065217 + } + }, + "qarinahV2": { + "deliveredTokens": 1997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.021739, + "recallAt10": 0.021739, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.060431, + "directRecallAt10": 0, + "supportingRecallAt10": 0.021739 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.021739, + "recallAt10": 0.021739, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.060431, + "directRecallAt10": 0, + "supportingRecallAt10": 0.021739 + } + }, + "oracle": { + "deliveredTokens": 1966, + "metrics": { + "recallAt1": 0.021739, + "recallAt5": 0.108696, + "recallAt10": 0.108696, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.773113, + "directRecallAt10": 0.8, + "supportingRecallAt10": 0.108696 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3966, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.065217, + "recallAt10": 0.065217, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.149572, + "directRecallAt10": 0, + "supportingRecallAt10": 0.065217 + } + }, + "balancedV1": { + "deliveredTokens": 3907, + "metrics": { + "recallAt1": 0.021739, + "recallAt5": 0.108696, + "recallAt10": 0.130435, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.686369, + "directRecallAt10": 0.6, + "supportingRecallAt10": 0.130435 + } + }, + "qarinahV2": { + "deliveredTokens": 3966, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.065217, + "recallAt10": 0.065217, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.149572, + "directRecallAt10": 0, + "supportingRecallAt10": 0.065217 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3966, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.065217, + "recallAt10": 0.065217, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.149572, + "directRecallAt10": 0, + "supportingRecallAt10": 0.065217 + } + }, + "oracle": { + "deliveredTokens": 3962, + "metrics": { + "recallAt1": 0.021739, + "recallAt5": 0.108696, + "recallAt10": 0.217391, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.217391 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.065217, + "recallAt10": 0.130435, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.315742, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.130435 + } + }, + "balancedV1": { + "deliveredTokens": 7880, + "metrics": { + "recallAt1": 0.021739, + "recallAt5": 0.108696, + "recallAt10": 0.217391, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.805031, + "directRecallAt10": 0.6, + "supportingRecallAt10": 0.217391 + } + }, + "qarinahV2": { + "deliveredTokens": 7994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.065217, + "recallAt10": 0.130435, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.315742, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.130435 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.065217, + "recallAt10": 0.130435, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.315742, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.130435 + } + }, + "oracle": { + "deliveredTokens": 7907, + "metrics": { + "recallAt1": 0.021739, + "recallAt5": 0.108696, + "recallAt10": 0.217391, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.217391 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.562184, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15819", + "repositorySequence": 95, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 10, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.1, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.305235, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0.1, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.305235, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.1, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.305235, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + }, + "oracle": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 5342, + "balancedV1": 3881, + "qarinahV2": 5342, + "qarinahV2NoGraph": 5342, + "oracle": 2982 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 487, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.1 + } + }, + "balancedV1": { + "deliveredTokens": 452, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 487, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 487, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.1 + } + }, + "oracle": { + "deliveredTokens": 375, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 994, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.1 + } + }, + "balancedV1": { + "deliveredTokens": 992, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 994, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 994, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.1 + } + }, + "oracle": { + "deliveredTokens": 816, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.3, + "recallAt10": 0.3, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.3 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1898, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.305235, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "balancedV1": { + "deliveredTokens": 1977, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1898, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.305235, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1898, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.305235, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "oracle": { + "deliveredTokens": 1897, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 0.7, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.800694, + "directRecallAt10": null, + "supportingRecallAt10": 0.7 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3965, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.2, + "recallAt10": 0.3, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.368856, + "directRecallAt10": null, + "supportingRecallAt10": 0.3 + } + }, + "balancedV1": { + "deliveredTokens": 3881, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3965, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.2, + "recallAt10": 0.3, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.368856, + "directRecallAt10": null, + "supportingRecallAt10": 0.3 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3965, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.2, + "recallAt10": 0.3, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.368856, + "directRecallAt10": null, + "supportingRecallAt10": 0.3 + } + }, + "oracle": { + "deliveredTokens": 2982, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7976, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.305235, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "balancedV1": { + "deliveredTokens": 7918, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7976, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.305235, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7976, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.305235, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "oracle": { + "deliveredTokens": 2982, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.531707, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15851", + "repositorySequence": 96, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 47, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.021277, + "recallAt5": 0.085106, + "recallAt10": 0.12766, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.654722, + "directRecallAt10": null, + "supportingRecallAt10": 0.12766 + }, + "balancedV1": { + "recallAt1": 0.021277, + "recallAt5": 0.106383, + "recallAt10": 0.148936, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.785916, + "directRecallAt10": null, + "supportingRecallAt10": 0.148936 + }, + "qarinahV2": { + "recallAt1": 0.021277, + "recallAt5": 0.085106, + "recallAt10": 0.12766, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.654722, + "directRecallAt10": null, + "supportingRecallAt10": 0.12766 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.021277, + "recallAt5": 0.085106, + "recallAt10": 0.12766, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.654722, + "directRecallAt10": null, + "supportingRecallAt10": 0.12766 + }, + "oracle": { + "recallAt1": 0.021277, + "recallAt5": 0.106383, + "recallAt10": 0.212766, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.212766 + } + }, + "volumes": { + "bm25Admitted": 5571, + "balancedV1": 4792, + "qarinahV2": 5571, + "qarinahV2NoGraph": 5571, + "oracle": 4062 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 454, + "metrics": { + "recallAt1": 0.021277, + "recallAt5": 0.021277, + "recallAt10": 0.021277, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.021277 + } + }, + "qarinahV2": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 408, + "metrics": { + "recallAt1": 0.021277, + "recallAt5": 0.042553, + "recallAt10": 0.042553, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.042553 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.021277, + "recallAt5": 0.021277, + "recallAt10": 0.021277, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.021277 + } + }, + "balancedV1": { + "deliveredTokens": 937, + "metrics": { + "recallAt1": 0.021277, + "recallAt5": 0.042553, + "recallAt10": 0.042553, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.042553 + } + }, + "qarinahV2": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.021277, + "recallAt5": 0.021277, + "recallAt10": 0.021277, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.021277 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.021277, + "recallAt5": 0.021277, + "recallAt10": 0.021277, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.021277 + } + }, + "oracle": { + "deliveredTokens": 904, + "metrics": { + "recallAt1": 0.021277, + "recallAt5": 0.06383, + "recallAt10": 0.06383, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.06383 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1987, + "metrics": { + "recallAt1": 0.021277, + "recallAt5": 0.042553, + "recallAt10": 0.042553, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.330138, + "directRecallAt10": null, + "supportingRecallAt10": 0.042553 + } + }, + "balancedV1": { + "deliveredTokens": 1998, + "metrics": { + "recallAt1": 0.021277, + "recallAt5": 0.085106, + "recallAt10": 0.085106, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.554143, + "directRecallAt10": null, + "supportingRecallAt10": 0.085106 + } + }, + "qarinahV2": { + "deliveredTokens": 1987, + "metrics": { + "recallAt1": 0.021277, + "recallAt5": 0.042553, + "recallAt10": 0.042553, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.330138, + "directRecallAt10": null, + "supportingRecallAt10": 0.042553 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1987, + "metrics": { + "recallAt1": 0.021277, + "recallAt5": 0.042553, + "recallAt10": 0.042553, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.330138, + "directRecallAt10": null, + "supportingRecallAt10": 0.042553 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.021277, + "recallAt5": 0.106383, + "recallAt10": 0.148936, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.800694, + "directRecallAt10": null, + "supportingRecallAt10": 0.148936 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3946, + "metrics": { + "recallAt1": 0.021277, + "recallAt5": 0.085106, + "recallAt10": 0.106383, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.588467, + "directRecallAt10": null, + "supportingRecallAt10": 0.106383 + } + }, + "balancedV1": { + "deliveredTokens": 3999, + "metrics": { + "recallAt1": 0.021277, + "recallAt5": 0.106383, + "recallAt10": 0.106383, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.106383 + } + }, + "qarinahV2": { + "deliveredTokens": 3946, + "metrics": { + "recallAt1": 0.021277, + "recallAt5": 0.085106, + "recallAt10": 0.106383, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.588467, + "directRecallAt10": null, + "supportingRecallAt10": 0.106383 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3946, + "metrics": { + "recallAt1": 0.021277, + "recallAt5": 0.085106, + "recallAt10": 0.106383, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.588467, + "directRecallAt10": null, + "supportingRecallAt10": 0.106383 + } + }, + "oracle": { + "deliveredTokens": 3982, + "metrics": { + "recallAt1": 0.021277, + "recallAt5": 0.106383, + "recallAt10": 0.212766, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.212766 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7948, + "metrics": { + "recallAt1": 0.021277, + "recallAt5": 0.085106, + "recallAt10": 0.12766, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.654722, + "directRecallAt10": null, + "supportingRecallAt10": 0.12766 + } + }, + "balancedV1": { + "deliveredTokens": 7995, + "metrics": { + "recallAt1": 0.021277, + "recallAt5": 0.106383, + "recallAt10": 0.148936, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.785916, + "directRecallAt10": null, + "supportingRecallAt10": 0.148936 + } + }, + "qarinahV2": { + "deliveredTokens": 7948, + "metrics": { + "recallAt1": 0.021277, + "recallAt5": 0.085106, + "recallAt10": 0.12766, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.654722, + "directRecallAt10": null, + "supportingRecallAt10": 0.12766 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7948, + "metrics": { + "recallAt1": 0.021277, + "recallAt5": 0.085106, + "recallAt10": 0.12766, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.654722, + "directRecallAt10": null, + "supportingRecallAt10": 0.12766 + } + }, + "oracle": { + "deliveredTokens": 7935, + "metrics": { + "recallAt1": 0.021277, + "recallAt5": 0.106383, + "recallAt10": 0.212766, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.212766 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "DIRECTLY_SUPPORTED", + "decision": "ACCEPT_DIRECT", + "score": 0.671875, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "CONSERVATIVE_DIRECT_THRESHOLD_MET" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15902", + "repositorySequence": 97, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 6, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.285714, + "recallAt10": 0.428571, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.36077, + "directRecallAt10": 1, + "supportingRecallAt10": 0.428571 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.088684, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.285714, + "recallAt10": 0.428571, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.36077, + "directRecallAt10": 1, + "supportingRecallAt10": 0.428571 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.285714, + "recallAt10": 0.428571, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.36077, + "directRecallAt10": 1, + "supportingRecallAt10": 0.428571 + }, + "oracle": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3454, + "balancedV1": 4025, + "qarinahV2": 3454, + "qarinahV2NoGraph": 3454, + "oracle": 2135 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 510, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.111907, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + } + }, + "balancedV1": { + "deliveredTokens": 470, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 510, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.111907, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 510, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.111907, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + } + }, + "oracle": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.428571, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.732694, + "directRecallAt10": 1, + "supportingRecallAt10": 0.428571 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.200591, + "directRecallAt10": 0, + "supportingRecallAt10": 0.285714 + } + }, + "balancedV1": { + "deliveredTokens": 996, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.088684, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2": { + "deliveredTokens": 994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.200591, + "directRecallAt10": 0, + "supportingRecallAt10": 0.285714 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.200591, + "directRecallAt10": 0, + "supportingRecallAt10": 0.285714 + } + }, + "oracle": { + "deliveredTokens": 846, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.571429, + "recallAt10": 0.571429, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.809082, + "directRecallAt10": 1, + "supportingRecallAt10": 0.571429 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1915, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.200591, + "directRecallAt10": 0, + "supportingRecallAt10": 0.285714 + } + }, + "balancedV1": { + "deliveredTokens": 1936, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.088684, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2": { + "deliveredTokens": 1915, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.200591, + "directRecallAt10": 0, + "supportingRecallAt10": 0.285714 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1915, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.200591, + "directRecallAt10": 0, + "supportingRecallAt10": 0.285714 + } + }, + "oracle": { + "deliveredTokens": 1667, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 0.857143, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.940877, + "directRecallAt10": 1, + "supportingRecallAt10": 0.857143 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3932, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.285714, + "recallAt10": 0.428571, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.36077, + "directRecallAt10": 1, + "supportingRecallAt10": 0.428571 + } + }, + "balancedV1": { + "deliveredTokens": 3991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.088684, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2": { + "deliveredTokens": 3932, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.285714, + "recallAt10": 0.428571, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.36077, + "directRecallAt10": 1, + "supportingRecallAt10": 0.428571 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3932, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.285714, + "recallAt10": 0.428571, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.36077, + "directRecallAt10": 1, + "supportingRecallAt10": 0.428571 + } + }, + "oracle": { + "deliveredTokens": 2135, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7889, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.285714, + "recallAt10": 0.428571, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.36077, + "directRecallAt10": 1, + "supportingRecallAt10": 0.428571 + } + }, + "balancedV1": { + "deliveredTokens": 7934, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.088684, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2": { + "deliveredTokens": 7889, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.285714, + "recallAt10": 0.428571, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.36077, + "directRecallAt10": 1, + "supportingRecallAt10": 0.428571 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7889, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.285714, + "recallAt10": 0.428571, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.36077, + "directRecallAt10": 1, + "supportingRecallAt10": 0.428571 + } + }, + "oracle": { + "deliveredTokens": 2135, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.375, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "NO_CODE_ENTITY_MATCH", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-15996", + "repositorySequence": 98, + "positiveUnderStructuralOracle": true, + "directRecords": 3, + "supportingRecords": 45, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.020833, + "recallAt5": 0.0625, + "recallAt10": 0.145833, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.678085, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.145833 + }, + "balancedV1": { + "recallAt1": 0.020833, + "recallAt5": 0.104167, + "recallAt10": 0.166667, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.907975, + "directRecallAt10": 1, + "supportingRecallAt10": 0.166667 + }, + "qarinahV2": { + "recallAt1": 0.020833, + "recallAt5": 0.0625, + "recallAt10": 0.145833, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.678085, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.145833 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.020833, + "recallAt5": 0.0625, + "recallAt10": 0.145833, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.678085, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.145833 + }, + "oracle": { + "recallAt1": 0.020833, + "recallAt5": 0.104167, + "recallAt10": 0.208333, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.208333 + } + }, + "volumes": { + "bm25Admitted": 4201, + "balancedV1": 4164, + "qarinahV2": 4201, + "qarinahV2NoGraph": 4201, + "oracle": 3536 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.020833, + "recallAt5": 0.020833, + "recallAt10": 0.020833, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.340699, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.020833 + } + }, + "balancedV1": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.020833, + "recallAt5": 0.020833, + "recallAt10": 0.020833, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.340699, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.020833 + } + }, + "qarinahV2": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.020833, + "recallAt5": 0.020833, + "recallAt10": 0.020833, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.340699, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.020833 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.020833, + "recallAt5": 0.020833, + "recallAt10": 0.020833, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.340699, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.020833 + } + }, + "oracle": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.020833, + "recallAt5": 0.020833, + "recallAt10": 0.020833, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.340699, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.020833 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.020833, + "recallAt5": 0.020833, + "recallAt10": 0.020833, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.340699, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.020833 + } + }, + "balancedV1": { + "deliveredTokens": 914, + "metrics": { + "recallAt1": 0.020833, + "recallAt5": 0.041667, + "recallAt10": 0.041667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.555657, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.041667 + } + }, + "qarinahV2": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.020833, + "recallAt5": 0.020833, + "recallAt10": 0.020833, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.340699, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.020833 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.020833, + "recallAt5": 0.020833, + "recallAt10": 0.020833, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.340699, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.020833 + } + }, + "oracle": { + "deliveredTokens": 918, + "metrics": { + "recallAt1": 0.020833, + "recallAt5": 0.041667, + "recallAt10": 0.041667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.555657, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.041667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1986, + "metrics": { + "recallAt1": 0.020833, + "recallAt5": 0.0625, + "recallAt10": 0.0625, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469135, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.0625 + } + }, + "balancedV1": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0.020833, + "recallAt5": 0.104167, + "recallAt10": 0.104167, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.803105, + "directRecallAt10": 1, + "supportingRecallAt10": 0.104167 + } + }, + "qarinahV2": { + "deliveredTokens": 1986, + "metrics": { + "recallAt1": 0.020833, + "recallAt5": 0.0625, + "recallAt10": 0.0625, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469135, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.0625 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1986, + "metrics": { + "recallAt1": 0.020833, + "recallAt5": 0.0625, + "recallAt10": 0.0625, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469135, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.0625 + } + }, + "oracle": { + "deliveredTokens": 1897, + "metrics": { + "recallAt1": 0.020833, + "recallAt5": 0.104167, + "recallAt10": 0.104167, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.81885, + "directRecallAt10": 1, + "supportingRecallAt10": 0.104167 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3964, + "metrics": { + "recallAt1": 0.020833, + "recallAt5": 0.0625, + "recallAt10": 0.145833, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.612429, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.145833 + } + }, + "balancedV1": { + "deliveredTokens": 3993, + "metrics": { + "recallAt1": 0.020833, + "recallAt5": 0.104167, + "recallAt10": 0.166667, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.907975, + "directRecallAt10": 1, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2": { + "deliveredTokens": 3964, + "metrics": { + "recallAt1": 0.020833, + "recallAt5": 0.0625, + "recallAt10": 0.145833, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.612429, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.145833 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3964, + "metrics": { + "recallAt1": 0.020833, + "recallAt5": 0.0625, + "recallAt10": 0.145833, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.612429, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.145833 + } + }, + "oracle": { + "deliveredTokens": 3898, + "metrics": { + "recallAt1": 0.020833, + "recallAt5": 0.104167, + "recallAt10": 0.208333, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.208333 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7929, + "metrics": { + "recallAt1": 0.020833, + "recallAt5": 0.0625, + "recallAt10": 0.145833, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.678085, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.145833 + } + }, + "balancedV1": { + "deliveredTokens": 7932, + "metrics": { + "recallAt1": 0.020833, + "recallAt5": 0.104167, + "recallAt10": 0.166667, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.907975, + "directRecallAt10": 1, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2": { + "deliveredTokens": 7929, + "metrics": { + "recallAt1": 0.020833, + "recallAt5": 0.0625, + "recallAt10": 0.145833, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.678085, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.145833 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7929, + "metrics": { + "recallAt1": 0.020833, + "recallAt5": 0.0625, + "recallAt10": 0.145833, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.678085, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.145833 + } + }, + "oracle": { + "deliveredTokens": 7983, + "metrics": { + "recallAt1": 0.020833, + "recallAt5": 0.104167, + "recallAt10": 0.208333, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.208333 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.489858, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-16041", + "repositorySequence": 99, + "positiveUnderStructuralOracle": true, + "directRecords": 2, + "supportingRecords": 6, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.286698, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.25 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.125, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.148104, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.125 + }, + "qarinahV2": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.286698, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.25 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.286698, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.25 + }, + "oracle": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 5756, + "balancedV1": 6351, + "qarinahV2": 5756, + "qarinahV2NoGraph": 5756, + "oracle": 2303 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 468, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.138594, + "directRecallAt10": 0, + "supportingRecallAt10": 0.125 + } + }, + "balancedV1": { + "deliveredTokens": 484, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 468, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.138594, + "directRecallAt10": 0, + "supportingRecallAt10": 0.125 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 468, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.138594, + "directRecallAt10": 0, + "supportingRecallAt10": 0.125 + } + }, + "oracle": { + "deliveredTokens": 505, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.375, + "recallAt10": 0.375, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.747408, + "directRecallAt10": 1, + "supportingRecallAt10": 0.375 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 931, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.346485, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 893, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 931, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.346485, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 931, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.346485, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 869, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.807097, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.25, + "recallAt10": 0.375, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.345638, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.375 + } + }, + "balancedV1": { + "deliveredTokens": 1925, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.25, + "recallAt10": 0.375, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.345638, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.375 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.25, + "recallAt10": 0.375, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.345638, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.375 + } + }, + "oracle": { + "deliveredTokens": 1835, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 0.875, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.956278, + "directRecallAt10": 1, + "supportingRecallAt10": 0.875 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 4000, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.286698, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 3990, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.125, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.148104, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.125 + } + }, + "qarinahV2": { + "deliveredTokens": 4000, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.286698, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 4000, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.286698, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 2303, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7889, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.286698, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 7956, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.125, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.148104, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.125 + } + }, + "qarinahV2": { + "deliveredTokens": 7889, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.286698, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7889, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.286698, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 2303, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.43956, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-16046", + "repositorySequence": 100, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 7, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "oracle": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3902, + "balancedV1": 3560, + "qarinahV2": 3902, + "qarinahV2NoGraph": 3902, + "oracle": 2512 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 423, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 477, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 423, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 423, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 439, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.448304, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 968, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 996, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 968, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 968, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 834, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.571429, + "recallAt10": 0.571429, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.704125, + "directRecallAt10": null, + "supportingRecallAt10": 0.571429 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1903, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1993, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1903, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1903, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 1914, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.571429, + "recallAt10": 0.571429, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.704125, + "directRecallAt10": null, + "supportingRecallAt10": 0.571429 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3902, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 3880, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3902, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3902, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 2512, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7982, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 7926, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7982, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7982, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 2512, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.535, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-16139", + "repositorySequence": 101, + "positiveUnderStructuralOracle": true, + "directRecords": 7, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.166667, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.214392, + "directRecallAt10": 0.142857, + "supportingRecallAt10": 0.166667 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.25, + "ndcgAt10": 0.069168, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + }, + "qarinahV2": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.166667, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.214392, + "directRecallAt10": 0.142857, + "supportingRecallAt10": 0.166667 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.166667, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.214392, + "directRecallAt10": 0.142857, + "supportingRecallAt10": 0.166667 + }, + "oracle": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.555556, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.555556 + } + }, + "volumes": { + "bm25Admitted": 4177, + "balancedV1": 5340, + "qarinahV2": 4177, + "qarinahV2NoGraph": 4177, + "oracle": 6715 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 486, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.244746, + "directRecallAt10": 0.142857, + "supportingRecallAt10": 0.111111 + } + }, + "balancedV1": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 486, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.244746, + "directRecallAt10": 0.142857, + "supportingRecallAt10": 0.111111 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 486, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.244746, + "directRecallAt10": 0.142857, + "supportingRecallAt10": 0.111111 + } + }, + "oracle": { + "deliveredTokens": 370, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.055556, + "recallAt10": 0.055556, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.253817, + "directRecallAt10": 0.142857, + "supportingRecallAt10": 0.055556 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 954, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.211514, + "directRecallAt10": 0.142857, + "supportingRecallAt10": 0.111111 + } + }, + "balancedV1": { + "deliveredTokens": 879, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 954, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.211514, + "directRecallAt10": 0.142857, + "supportingRecallAt10": 0.111111 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 954, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.211514, + "directRecallAt10": 0.142857, + "supportingRecallAt10": 0.111111 + } + }, + "oracle": { + "deliveredTokens": 984, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.413957, + "directRecallAt10": 0.285714, + "supportingRecallAt10": 0.111111 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1925, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.137986, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + } + }, + "balancedV1": { + "deliveredTokens": 1982, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.25, + "ndcgAt10": 0.134627, + "directRecallAt10": 0.142857, + "supportingRecallAt10": 0.111111 + } + }, + "qarinahV2": { + "deliveredTokens": 1925, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.137986, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1925, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.137986, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + } + }, + "oracle": { + "deliveredTokens": 1926, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.333333, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.778505, + "directRecallAt10": 0.714286, + "supportingRecallAt10": 0.333333 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3997, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.137986, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + } + }, + "balancedV1": { + "deliveredTokens": 3980, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.25, + "ndcgAt10": 0.069168, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + } + }, + "qarinahV2": { + "deliveredTokens": 3997, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.137986, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3997, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.137986, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + } + }, + "oracle": { + "deliveredTokens": 3994, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.444444, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.893671, + "directRecallAt10": 0.857143, + "supportingRecallAt10": 0.444444 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7893, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.166667, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.214392, + "directRecallAt10": 0.142857, + "supportingRecallAt10": 0.166667 + } + }, + "balancedV1": { + "deliveredTokens": 7959, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.25, + "ndcgAt10": 0.069168, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + } + }, + "qarinahV2": { + "deliveredTokens": 7893, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.166667, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.214392, + "directRecallAt10": 0.142857, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7893, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.166667, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.214392, + "directRecallAt10": 0.142857, + "supportingRecallAt10": 0.166667 + } + }, + "oracle": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.555556, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.555556 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.495806, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-16229", + "repositorySequence": 102, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 8, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.3, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.14995, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.061852, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.3, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.14995, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.3, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.14995, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + }, + "oracle": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4672, + "balancedV1": 4914, + "qarinahV2": 4672, + "qarinahV2NoGraph": 4672, + "oracle": 2676 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 496, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.100876, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + } + }, + "balancedV1": { + "deliveredTokens": 484, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.100876, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + } + }, + "qarinahV2": { + "deliveredTokens": 496, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.100876, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 496, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.100876, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + } + }, + "oracle": { + "deliveredTokens": 468, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.479655, + "directRecallAt10": 1, + "supportingRecallAt10": 0.111111 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 938, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 943, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.079943, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + } + }, + "qarinahV2": { + "deliveredTokens": 938, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 938, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 963, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.660474, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1914, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.061852, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + } + }, + "balancedV1": { + "deliveredTokens": 1950, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.068859, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + } + }, + "qarinahV2": { + "deliveredTokens": 1914, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.061852, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1914, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.061852, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + } + }, + "oracle": { + "deliveredTokens": 1955, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 0.777778, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.901432, + "directRecallAt10": 1, + "supportingRecallAt10": 0.777778 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3980, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.3, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.151863, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 3974, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.222222, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.109982, + "directRecallAt10": 0, + "supportingRecallAt10": 0.222222 + } + }, + "qarinahV2": { + "deliveredTokens": 3980, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.3, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.151863, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3980, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.3, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.151863, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 2676, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7947, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.3, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.14995, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 7894, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.061852, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + } + }, + "qarinahV2": { + "deliveredTokens": 7947, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.3, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.14995, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7947, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.3, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.14995, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 2676, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.517391, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-16255", + "repositorySequence": 103, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 14, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.248908, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.142857, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.139618, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.248908, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.248908, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + }, + "oracle": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.714286, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.714286 + } + }, + "volumes": { + "bm25Admitted": 9000, + "balancedV1": 6106, + "qarinahV2": 9000, + "qarinahV2NoGraph": 9000, + "oracle": 3935 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 505, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 492, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 961, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.071429 + } + }, + "balancedV1": { + "deliveredTokens": 964, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 961, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.071429 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 961, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.071429 + } + }, + "oracle": { + "deliveredTokens": 909, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.214286, + "recallAt10": 0.214286, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.214286 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1986, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.248908, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "balancedV1": { + "deliveredTokens": 1991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1986, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.248908, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1986, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.248908, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "oracle": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3876, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.248908, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "balancedV1": { + "deliveredTokens": 3975, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.142857, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.142795, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2": { + "deliveredTokens": 3876, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.248908, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3876, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.248908, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "oracle": { + "deliveredTokens": 3935, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.714286, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.714286 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7961, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.248908, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "balancedV1": { + "deliveredTokens": 7987, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.142857, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.139618, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2": { + "deliveredTokens": 7961, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.248908, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7961, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.248908, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "oracle": { + "deliveredTokens": 5164, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.714286, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.714286 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.589643, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-16379", + "repositorySequence": 104, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.31488, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.181818, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.151762, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.31488, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.31488, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 9084, + "balancedV1": 6684, + "qarinahV2": 9084, + "qarinahV2NoGraph": 9084, + "oracle": 2953 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 508, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 456, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2": { + "deliveredTokens": 508, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 508, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 384, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 929, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 992, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1974, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 1894, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.204834, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 1974, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1974, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 1943, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.727273, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.870125, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.31488, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 3976, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.085143, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2": { + "deliveredTokens": 3971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.31488, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3971, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.31488, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 3180, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7970, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.31488, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 7994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.181818, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.151762, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 7970, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.31488, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7970, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.31488, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 3180, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.557703, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-16400", + "repositorySequence": 105, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 15, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.334051, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.066667, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.069431, + "directRecallAt10": null, + "supportingRecallAt10": 0.066667 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.334051, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.334051, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + }, + "oracle": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "volumes": { + "bm25Admitted": 6707, + "balancedV1": 6299, + "qarinahV2": 6707, + "qarinahV2NoGraph": 6707, + "oracle": 4237 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 478, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 484, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 478, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 478, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 492, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.133333, + "recallAt10": 0.133333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.133333 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 931, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.066667, + "recallAt10": 0.066667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.066667 + } + }, + "balancedV1": { + "deliveredTokens": 968, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.066667, + "recallAt10": 0.066667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.066667 + } + }, + "qarinahV2": { + "deliveredTokens": 931, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.066667, + "recallAt10": 0.066667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.066667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 931, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.066667, + "recallAt10": 0.066667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.066667 + } + }, + "oracle": { + "deliveredTokens": 909, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1906, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.066667, + "recallAt10": 0.066667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.066667 + } + }, + "balancedV1": { + "deliveredTokens": 1925, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1906, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.066667, + "recallAt10": 0.066667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.066667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1906, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.066667, + "recallAt10": 0.066667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.066667 + } + }, + "oracle": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.266667, + "recallAt10": 0.266667, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.266667 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3938, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.266667, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.407415, + "directRecallAt10": null, + "supportingRecallAt10": 0.266667 + } + }, + "balancedV1": { + "deliveredTokens": 3953, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3938, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.266667, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.407415, + "directRecallAt10": null, + "supportingRecallAt10": 0.266667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3938, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.266667, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.407415, + "directRecallAt10": null, + "supportingRecallAt10": 0.266667 + } + }, + "oracle": { + "deliveredTokens": 3948, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.6, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.936379, + "directRecallAt10": null, + "supportingRecallAt10": 0.6 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7923, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.334051, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "balancedV1": { + "deliveredTokens": 7902, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.066667, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.069431, + "directRecallAt10": null, + "supportingRecallAt10": 0.066667 + } + }, + "qarinahV2": { + "deliveredTokens": 7923, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.334051, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7923, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.334051, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "oracle": { + "deliveredTokens": 5617, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.441308, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-16408", + "repositorySequence": 106, + "positiveUnderStructuralOracle": true, + "directRecords": 3, + "supportingRecords": 46, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.020408, + "recallAt5": 0.081633, + "recallAt10": 0.142857, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.401379, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + }, + "balancedV1": { + "recallAt1": 0.020408, + "recallAt5": 0.102041, + "recallAt10": 0.163265, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.448981, + "directRecallAt10": 0, + "supportingRecallAt10": 0.163265 + }, + "qarinahV2": { + "recallAt1": 0.020408, + "recallAt5": 0.081633, + "recallAt10": 0.142857, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.401379, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.020408, + "recallAt5": 0.081633, + "recallAt10": 0.142857, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.401379, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + }, + "oracle": { + "recallAt1": 0.020408, + "recallAt5": 0.102041, + "recallAt10": 0.204082, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.204082 + } + }, + "volumes": { + "bm25Admitted": 8238, + "balancedV1": 5592, + "qarinahV2": 8238, + "qarinahV2NoGraph": 8238, + "oracle": 4098 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 507, + "metrics": { + "recallAt1": 0.020408, + "recallAt5": 0.020408, + "recallAt10": 0.020408, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.113566, + "directRecallAt10": 0, + "supportingRecallAt10": 0.020408 + } + }, + "balancedV1": { + "deliveredTokens": 464, + "metrics": { + "recallAt1": 0.020408, + "recallAt5": 0.040816, + "recallAt10": 0.040816, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.185219, + "directRecallAt10": 0, + "supportingRecallAt10": 0.040816 + } + }, + "qarinahV2": { + "deliveredTokens": 507, + "metrics": { + "recallAt1": 0.020408, + "recallAt5": 0.020408, + "recallAt10": 0.020408, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.113566, + "directRecallAt10": 0, + "supportingRecallAt10": 0.020408 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 507, + "metrics": { + "recallAt1": 0.020408, + "recallAt5": 0.020408, + "recallAt10": 0.020408, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.113566, + "directRecallAt10": 0, + "supportingRecallAt10": 0.020408 + } + }, + "oracle": { + "deliveredTokens": 454, + "metrics": { + "recallAt1": 0.020408, + "recallAt5": 0.040816, + "recallAt10": 0.040816, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.412352, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.040816 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 997, + "metrics": { + "recallAt1": 0.020408, + "recallAt5": 0.061224, + "recallAt10": 0.061224, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.242002, + "directRecallAt10": 0, + "supportingRecallAt10": 0.061224 + } + }, + "balancedV1": { + "deliveredTokens": 965, + "metrics": { + "recallAt1": 0.020408, + "recallAt5": 0.081633, + "recallAt10": 0.081633, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.290912, + "directRecallAt10": 0, + "supportingRecallAt10": 0.081633 + } + }, + "qarinahV2": { + "deliveredTokens": 997, + "metrics": { + "recallAt1": 0.020408, + "recallAt5": 0.061224, + "recallAt10": 0.061224, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.242002, + "directRecallAt10": 0, + "supportingRecallAt10": 0.061224 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 997, + "metrics": { + "recallAt1": 0.020408, + "recallAt5": 0.061224, + "recallAt10": 0.061224, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.242002, + "directRecallAt10": 0, + "supportingRecallAt10": 0.061224 + } + }, + "oracle": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.020408, + "recallAt5": 0.020408, + "recallAt10": 0.020408, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.340699, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.020408 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0.020408, + "recallAt5": 0.081633, + "recallAt10": 0.081633, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.285936, + "directRecallAt10": 0, + "supportingRecallAt10": 0.081633 + } + }, + "balancedV1": { + "deliveredTokens": 1945, + "metrics": { + "recallAt1": 0.020408, + "recallAt5": 0.061224, + "recallAt10": 0.061224, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.242002, + "directRecallAt10": 0, + "supportingRecallAt10": 0.061224 + } + }, + "qarinahV2": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0.020408, + "recallAt5": 0.081633, + "recallAt10": 0.081633, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.285936, + "directRecallAt10": 0, + "supportingRecallAt10": 0.081633 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0.020408, + "recallAt5": 0.081633, + "recallAt10": 0.081633, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.285936, + "directRecallAt10": 0, + "supportingRecallAt10": 0.081633 + } + }, + "oracle": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0.020408, + "recallAt5": 0.081633, + "recallAt10": 0.081633, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.774917, + "directRecallAt10": 1, + "supportingRecallAt10": 0.081633 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3969, + "metrics": { + "recallAt1": 0.020408, + "recallAt5": 0.081633, + "recallAt10": 0.102041, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.328768, + "directRecallAt10": 0, + "supportingRecallAt10": 0.102041 + } + }, + "balancedV1": { + "deliveredTokens": 3982, + "metrics": { + "recallAt1": 0.020408, + "recallAt5": 0.102041, + "recallAt10": 0.142857, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.411125, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2": { + "deliveredTokens": 3969, + "metrics": { + "recallAt1": 0.020408, + "recallAt5": 0.081633, + "recallAt10": 0.102041, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.328768, + "directRecallAt10": 0, + "supportingRecallAt10": 0.102041 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3969, + "metrics": { + "recallAt1": 0.020408, + "recallAt5": 0.081633, + "recallAt10": 0.102041, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.328768, + "directRecallAt10": 0, + "supportingRecallAt10": 0.102041 + } + }, + "oracle": { + "deliveredTokens": 3927, + "metrics": { + "recallAt1": 0.020408, + "recallAt5": 0.102041, + "recallAt10": 0.183673, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.967172, + "directRecallAt10": 1, + "supportingRecallAt10": 0.183673 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7983, + "metrics": { + "recallAt1": 0.020408, + "recallAt5": 0.081633, + "recallAt10": 0.163265, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.434207, + "directRecallAt10": 0, + "supportingRecallAt10": 0.163265 + } + }, + "balancedV1": { + "deliveredTokens": 7909, + "metrics": { + "recallAt1": 0.020408, + "recallAt5": 0.102041, + "recallAt10": 0.163265, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.448981, + "directRecallAt10": 0, + "supportingRecallAt10": 0.163265 + } + }, + "qarinahV2": { + "deliveredTokens": 7983, + "metrics": { + "recallAt1": 0.020408, + "recallAt5": 0.081633, + "recallAt10": 0.163265, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.434207, + "directRecallAt10": 0, + "supportingRecallAt10": 0.163265 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7983, + "metrics": { + "recallAt1": 0.020408, + "recallAt5": 0.081633, + "recallAt10": 0.163265, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.434207, + "directRecallAt10": 0, + "supportingRecallAt10": 0.163265 + } + }, + "oracle": { + "deliveredTokens": 7899, + "metrics": { + "recallAt1": 0.020408, + "recallAt5": 0.102041, + "recallAt10": 0.204082, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.204082 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.579221, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-16527", + "repositorySequence": 107, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 16, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.0625, + "recallAt10": 0.3125, + "precisionAt5": 0.2, + "precisionAt10": 0.5, + "reciprocalRank": 0.25, + "ndcgAt10": 0.379603, + "directRecallAt10": null, + "supportingRecallAt10": 0.3125 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.125, + "recallAt10": 0.3125, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.432957, + "directRecallAt10": null, + "supportingRecallAt10": 0.3125 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.0625, + "recallAt10": 0.3125, + "precisionAt5": 0.2, + "precisionAt10": 0.5, + "reciprocalRank": 0.25, + "ndcgAt10": 0.379603, + "directRecallAt10": null, + "supportingRecallAt10": 0.3125 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.0625, + "recallAt10": 0.3125, + "precisionAt5": 0.2, + "precisionAt10": 0.5, + "reciprocalRank": 0.25, + "ndcgAt10": 0.379603, + "directRecallAt10": null, + "supportingRecallAt10": 0.3125 + }, + "oracle": { + "recallAt1": 0.0625, + "recallAt5": 0.3125, + "recallAt10": 0.625, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.625 + } + }, + "volumes": { + "bm25Admitted": 5123, + "balancedV1": 3454, + "qarinahV2": 5123, + "qarinahV2NoGraph": 5123, + "oracle": 4606 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 494, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 399, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.0625, + "recallAt10": 0.0625, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.0625 + } + }, + "qarinahV2": { + "deliveredTokens": 494, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 494, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 492, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 920, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 919, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.248908, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + }, + "qarinahV2": { + "deliveredTokens": 920, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 920, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 909, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.1875, + "recallAt10": 0.1875, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.1875 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1999, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.0625, + "recallAt10": 0.0625, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.094788, + "directRecallAt10": null, + "supportingRecallAt10": 0.0625 + } + }, + "balancedV1": { + "deliveredTokens": 1998, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.0625, + "recallAt10": 0.0625, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.0625 + } + }, + "qarinahV2": { + "deliveredTokens": 1999, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.0625, + "recallAt10": 0.0625, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.094788, + "directRecallAt10": null, + "supportingRecallAt10": 0.0625 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1999, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.0625, + "recallAt10": 0.0625, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.094788, + "directRecallAt10": null, + "supportingRecallAt10": 0.0625 + } + }, + "oracle": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.0625, + "recallAt10": 0.3125, + "precisionAt5": 0.2, + "precisionAt10": 0.5, + "reciprocalRank": 0.25, + "ndcgAt10": 0.382236, + "directRecallAt10": null, + "supportingRecallAt10": 0.3125 + } + }, + "balancedV1": { + "deliveredTokens": 3889, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.125, + "recallAt10": 0.3125, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.432957, + "directRecallAt10": null, + "supportingRecallAt10": 0.3125 + } + }, + "qarinahV2": { + "deliveredTokens": 3997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.0625, + "recallAt10": 0.3125, + "precisionAt5": 0.2, + "precisionAt10": 0.5, + "reciprocalRank": 0.25, + "ndcgAt10": 0.382236, + "directRecallAt10": null, + "supportingRecallAt10": 0.3125 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.0625, + "recallAt10": 0.3125, + "precisionAt5": 0.2, + "precisionAt10": 0.5, + "reciprocalRank": 0.25, + "ndcgAt10": 0.382236, + "directRecallAt10": null, + "supportingRecallAt10": 0.3125 + } + }, + "oracle": { + "deliveredTokens": 3969, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.3125, + "recallAt10": 0.625, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.625 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7944, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.0625, + "recallAt10": 0.3125, + "precisionAt5": 0.2, + "precisionAt10": 0.5, + "reciprocalRank": 0.25, + "ndcgAt10": 0.379603, + "directRecallAt10": null, + "supportingRecallAt10": 0.3125 + } + }, + "balancedV1": { + "deliveredTokens": 7926, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.125, + "recallAt10": 0.3125, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.432957, + "directRecallAt10": null, + "supportingRecallAt10": 0.3125 + } + }, + "qarinahV2": { + "deliveredTokens": 7944, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.0625, + "recallAt10": 0.3125, + "precisionAt5": 0.2, + "precisionAt10": 0.5, + "reciprocalRank": 0.25, + "ndcgAt10": 0.379603, + "directRecallAt10": null, + "supportingRecallAt10": 0.3125 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7944, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.0625, + "recallAt10": 0.3125, + "precisionAt5": 0.2, + "precisionAt10": 0.5, + "reciprocalRank": 0.25, + "ndcgAt10": 0.379603, + "directRecallAt10": null, + "supportingRecallAt10": 0.3125 + } + }, + "oracle": { + "deliveredTokens": 6275, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.3125, + "recallAt10": 0.625, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.625 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.6, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-16595", + "repositorySequence": 108, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 50, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.02, + "recallAt5": 0.08, + "recallAt10": 0.16, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.851236, + "directRecallAt10": null, + "supportingRecallAt10": 0.16 + }, + "balancedV1": { + "recallAt1": 0.02, + "recallAt5": 0.1, + "recallAt10": 0.18, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.926636, + "directRecallAt10": null, + "supportingRecallAt10": 0.18 + }, + "qarinahV2": { + "recallAt1": 0.02, + "recallAt5": 0.08, + "recallAt10": 0.16, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.851236, + "directRecallAt10": null, + "supportingRecallAt10": 0.16 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.02, + "recallAt5": 0.08, + "recallAt10": 0.16, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.851236, + "directRecallAt10": null, + "supportingRecallAt10": 0.16 + }, + "oracle": { + "recallAt1": 0.02, + "recallAt5": 0.1, + "recallAt10": 0.2, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "volumes": { + "bm25Admitted": 5345, + "balancedV1": 4246, + "qarinahV2": 5345, + "qarinahV2NoGraph": 5345, + "oracle": 4062 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.02, + "recallAt5": 0.02, + "recallAt10": 0.02, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.02 + } + }, + "balancedV1": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.02, + "recallAt5": 0.02, + "recallAt10": 0.02, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.02 + } + }, + "qarinahV2": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.02, + "recallAt5": 0.02, + "recallAt10": 0.02, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.02 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.02, + "recallAt5": 0.02, + "recallAt10": 0.02, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.02 + } + }, + "oracle": { + "deliveredTokens": 408, + "metrics": { + "recallAt1": 0.02, + "recallAt5": 0.04, + "recallAt10": 0.04, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.04 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 959, + "metrics": { + "recallAt1": 0.02, + "recallAt5": 0.06, + "recallAt10": 0.06, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.06 + } + }, + "balancedV1": { + "deliveredTokens": 907, + "metrics": { + "recallAt1": 0.02, + "recallAt5": 0.04, + "recallAt10": 0.04, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.04 + } + }, + "qarinahV2": { + "deliveredTokens": 959, + "metrics": { + "recallAt1": 0.02, + "recallAt5": 0.06, + "recallAt10": 0.06, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.06 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 959, + "metrics": { + "recallAt1": 0.02, + "recallAt5": 0.06, + "recallAt10": 0.06, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.06 + } + }, + "oracle": { + "deliveredTokens": 904, + "metrics": { + "recallAt1": 0.02, + "recallAt5": 0.06, + "recallAt10": 0.06, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.06 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1982, + "metrics": { + "recallAt1": 0.02, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.1 + } + }, + "balancedV1": { + "deliveredTokens": 1959, + "metrics": { + "recallAt1": 0.02, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.1 + } + }, + "qarinahV2": { + "deliveredTokens": 1982, + "metrics": { + "recallAt1": 0.02, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1982, + "metrics": { + "recallAt1": 0.02, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.1 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.02, + "recallAt5": 0.1, + "recallAt10": 0.14, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.800694, + "directRecallAt10": null, + "supportingRecallAt10": 0.14 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3999, + "metrics": { + "recallAt1": 0.02, + "recallAt5": 0.08, + "recallAt10": 0.12, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.715551, + "directRecallAt10": null, + "supportingRecallAt10": 0.12 + } + }, + "balancedV1": { + "deliveredTokens": 3923, + "metrics": { + "recallAt1": 0.02, + "recallAt5": 0.1, + "recallAt10": 0.16, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.863015, + "directRecallAt10": null, + "supportingRecallAt10": 0.16 + } + }, + "qarinahV2": { + "deliveredTokens": 3999, + "metrics": { + "recallAt1": 0.02, + "recallAt5": 0.08, + "recallAt10": 0.12, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.715551, + "directRecallAt10": null, + "supportingRecallAt10": 0.12 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3999, + "metrics": { + "recallAt1": 0.02, + "recallAt5": 0.08, + "recallAt10": 0.12, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.715551, + "directRecallAt10": null, + "supportingRecallAt10": 0.12 + } + }, + "oracle": { + "deliveredTokens": 3982, + "metrics": { + "recallAt1": 0.02, + "recallAt5": 0.1, + "recallAt10": 0.2, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7974, + "metrics": { + "recallAt1": 0.02, + "recallAt5": 0.08, + "recallAt10": 0.16, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.851236, + "directRecallAt10": null, + "supportingRecallAt10": 0.16 + } + }, + "balancedV1": { + "deliveredTokens": 7962, + "metrics": { + "recallAt1": 0.02, + "recallAt5": 0.1, + "recallAt10": 0.18, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.926636, + "directRecallAt10": null, + "supportingRecallAt10": 0.18 + } + }, + "qarinahV2": { + "deliveredTokens": 7974, + "metrics": { + "recallAt1": 0.02, + "recallAt5": 0.08, + "recallAt10": 0.16, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.851236, + "directRecallAt10": null, + "supportingRecallAt10": 0.16 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7974, + "metrics": { + "recallAt1": 0.02, + "recallAt5": 0.08, + "recallAt10": 0.16, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.851236, + "directRecallAt10": null, + "supportingRecallAt10": 0.16 + } + }, + "oracle": { + "deliveredTokens": 7935, + "metrics": { + "recallAt1": 0.02, + "recallAt5": 0.1, + "recallAt10": 0.2, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.542747, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-16816", + "repositorySequence": 109, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 17, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.058824, + "recallAt5": 0.117647, + "recallAt10": 0.294118, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.568004, + "directRecallAt10": null, + "supportingRecallAt10": 0.294118 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.117647, + "recallAt10": 0.235294, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.334709, + "directRecallAt10": null, + "supportingRecallAt10": 0.235294 + }, + "qarinahV2": { + "recallAt1": 0.058824, + "recallAt5": 0.117647, + "recallAt10": 0.294118, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.568004, + "directRecallAt10": null, + "supportingRecallAt10": 0.294118 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.058824, + "recallAt5": 0.117647, + "recallAt10": 0.294118, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.568004, + "directRecallAt10": null, + "supportingRecallAt10": 0.294118 + }, + "oracle": { + "recallAt1": 0.058824, + "recallAt5": 0.294118, + "recallAt10": 0.588235, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.588235 + } + }, + "volumes": { + "bm25Admitted": 8114, + "balancedV1": 7182, + "qarinahV2": 8114, + "qarinahV2NoGraph": 8114, + "oracle": 4541 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 453, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.058824, + "recallAt10": 0.058824, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.058824 + } + }, + "balancedV1": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 453, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.058824, + "recallAt10": 0.058824, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.058824 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 453, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.058824, + "recallAt10": 0.058824, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.058824 + } + }, + "oracle": { + "deliveredTokens": 492, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.117647, + "recallAt10": 0.117647, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.117647 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 964, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.058824, + "recallAt10": 0.058824, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.058824 + } + }, + "balancedV1": { + "deliveredTokens": 912, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.058824, + "recallAt10": 0.058824, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.058824 + } + }, + "qarinahV2": { + "deliveredTokens": 964, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.058824, + "recallAt10": 0.058824, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.058824 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 964, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.058824, + "recallAt10": 0.058824, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.058824 + } + }, + "oracle": { + "deliveredTokens": 909, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.176471, + "recallAt10": 0.176471, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.176471 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1909, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.117647, + "recallAt10": 0.117647, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.117647 + } + }, + "balancedV1": { + "deliveredTokens": 1999, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.117647, + "recallAt10": 0.117647, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.204834, + "directRecallAt10": null, + "supportingRecallAt10": 0.117647 + } + }, + "qarinahV2": { + "deliveredTokens": 1909, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.117647, + "recallAt10": 0.117647, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.117647 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1909, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.117647, + "recallAt10": 0.117647, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.117647 + } + }, + "oracle": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.235294, + "recallAt10": 0.235294, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.235294 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3965, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.117647, + "recallAt10": 0.176471, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.437352, + "directRecallAt10": null, + "supportingRecallAt10": 0.176471 + } + }, + "balancedV1": { + "deliveredTokens": 3910, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.117647, + "recallAt10": 0.176471, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.278198, + "directRecallAt10": null, + "supportingRecallAt10": 0.176471 + } + }, + "qarinahV2": { + "deliveredTokens": 3965, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.117647, + "recallAt10": 0.176471, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.437352, + "directRecallAt10": null, + "supportingRecallAt10": 0.176471 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3965, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.117647, + "recallAt10": 0.176471, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.437352, + "directRecallAt10": null, + "supportingRecallAt10": 0.176471 + } + }, + "oracle": { + "deliveredTokens": 3904, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.294118, + "recallAt10": 0.588235, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.588235 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7938, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.117647, + "recallAt10": 0.294118, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.568004, + "directRecallAt10": null, + "supportingRecallAt10": 0.294118 + } + }, + "balancedV1": { + "deliveredTokens": 7947, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.117647, + "recallAt10": 0.235294, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.334709, + "directRecallAt10": null, + "supportingRecallAt10": 0.235294 + } + }, + "qarinahV2": { + "deliveredTokens": 7938, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.117647, + "recallAt10": 0.294118, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.568004, + "directRecallAt10": null, + "supportingRecallAt10": 0.294118 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7938, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.117647, + "recallAt10": 0.294118, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.568004, + "directRecallAt10": null, + "supportingRecallAt10": 0.294118 + } + }, + "oracle": { + "deliveredTokens": 6461, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.294118, + "recallAt10": 0.588235, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.588235 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.468478, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-16820", + "repositorySequence": 110, + "positiveUnderStructuralOracle": true, + "directRecords": 2, + "supportingRecords": 49, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.019608, + "recallAt5": 0.058824, + "recallAt10": 0.117647, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.355941, + "directRecallAt10": 0, + "supportingRecallAt10": 0.117647 + }, + "balancedV1": { + "recallAt1": 0.019608, + "recallAt5": 0.098039, + "recallAt10": 0.156863, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.499433, + "directRecallAt10": 0, + "supportingRecallAt10": 0.156863 + }, + "qarinahV2": { + "recallAt1": 0.019608, + "recallAt5": 0.058824, + "recallAt10": 0.117647, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.355941, + "directRecallAt10": 0, + "supportingRecallAt10": 0.117647 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.019608, + "recallAt5": 0.058824, + "recallAt10": 0.117647, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.355941, + "directRecallAt10": 0, + "supportingRecallAt10": 0.117647 + }, + "oracle": { + "recallAt1": 0.019608, + "recallAt5": 0.098039, + "recallAt10": 0.196078, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.196078 + } + }, + "volumes": { + "bm25Admitted": 2723, + "balancedV1": 3608, + "qarinahV2": 2723, + "qarinahV2NoGraph": 2723, + "oracle": 3108 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 454, + "metrics": { + "recallAt1": 0.019608, + "recallAt5": 0.019608, + "recallAt10": 0.019608, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.128116, + "directRecallAt10": 0, + "supportingRecallAt10": 0.019608 + } + }, + "balancedV1": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.019608, + "recallAt5": 0.019608, + "recallAt10": 0.019608, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.128116, + "directRecallAt10": 0, + "supportingRecallAt10": 0.019608 + } + }, + "qarinahV2": { + "deliveredTokens": 454, + "metrics": { + "recallAt1": 0.019608, + "recallAt5": 0.019608, + "recallAt10": 0.019608, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.128116, + "directRecallAt10": 0, + "supportingRecallAt10": 0.019608 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 454, + "metrics": { + "recallAt1": 0.019608, + "recallAt5": 0.019608, + "recallAt10": 0.019608, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.128116, + "directRecallAt10": 0, + "supportingRecallAt10": 0.019608 + } + }, + "oracle": { + "deliveredTokens": 504, + "metrics": { + "recallAt1": 0.019608, + "recallAt5": 0.039216, + "recallAt10": 0.039216, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.626845, + "directRecallAt10": 1, + "supportingRecallAt10": 0.039216 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 971, + "metrics": { + "recallAt1": 0.019608, + "recallAt5": 0.058824, + "recallAt10": 0.058824, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.232855, + "directRecallAt10": 0, + "supportingRecallAt10": 0.058824 + } + }, + "balancedV1": { + "deliveredTokens": 947, + "metrics": { + "recallAt1": 0.019608, + "recallAt5": 0.039216, + "recallAt10": 0.039216, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.192174, + "directRecallAt10": 0, + "supportingRecallAt10": 0.039216 + } + }, + "qarinahV2": { + "deliveredTokens": 971, + "metrics": { + "recallAt1": 0.019608, + "recallAt5": 0.058824, + "recallAt10": 0.058824, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.232855, + "directRecallAt10": 0, + "supportingRecallAt10": 0.058824 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 971, + "metrics": { + "recallAt1": 0.019608, + "recallAt5": 0.058824, + "recallAt10": 0.058824, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.232855, + "directRecallAt10": 0, + "supportingRecallAt10": 0.058824 + } + }, + "oracle": { + "deliveredTokens": 912, + "metrics": { + "recallAt1": 0.019608, + "recallAt5": 0.078431, + "recallAt10": 0.078431, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.74608, + "directRecallAt10": 1, + "supportingRecallAt10": 0.078431 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1947, + "metrics": { + "recallAt1": 0.019608, + "recallAt5": 0.058824, + "recallAt10": 0.078431, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.278491, + "directRecallAt10": 0, + "supportingRecallAt10": 0.078431 + } + }, + "balancedV1": { + "deliveredTokens": 1951, + "metrics": { + "recallAt1": 0.019608, + "recallAt5": 0.078431, + "recallAt10": 0.078431, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.322569, + "directRecallAt10": 0, + "supportingRecallAt10": 0.078431 + } + }, + "qarinahV2": { + "deliveredTokens": 1947, + "metrics": { + "recallAt1": 0.019608, + "recallAt5": 0.058824, + "recallAt10": 0.078431, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.278491, + "directRecallAt10": 0, + "supportingRecallAt10": 0.078431 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1947, + "metrics": { + "recallAt1": 0.019608, + "recallAt5": 0.058824, + "recallAt10": 0.078431, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.278491, + "directRecallAt10": 0, + "supportingRecallAt10": 0.078431 + } + }, + "oracle": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0.019608, + "recallAt5": 0.098039, + "recallAt10": 0.137255, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.883983, + "directRecallAt10": 1, + "supportingRecallAt10": 0.137255 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3922, + "metrics": { + "recallAt1": 0.019608, + "recallAt5": 0.058824, + "recallAt10": 0.117647, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.355941, + "directRecallAt10": 0, + "supportingRecallAt10": 0.117647 + } + }, + "balancedV1": { + "deliveredTokens": 3945, + "metrics": { + "recallAt1": 0.019608, + "recallAt5": 0.098039, + "recallAt10": 0.156863, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.499433, + "directRecallAt10": 0, + "supportingRecallAt10": 0.156863 + } + }, + "qarinahV2": { + "deliveredTokens": 3922, + "metrics": { + "recallAt1": 0.019608, + "recallAt5": 0.058824, + "recallAt10": 0.117647, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.355941, + "directRecallAt10": 0, + "supportingRecallAt10": 0.117647 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3922, + "metrics": { + "recallAt1": 0.019608, + "recallAt5": 0.058824, + "recallAt10": 0.117647, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.355941, + "directRecallAt10": 0, + "supportingRecallAt10": 0.117647 + } + }, + "oracle": { + "deliveredTokens": 3932, + "metrics": { + "recallAt1": 0.019608, + "recallAt5": 0.098039, + "recallAt10": 0.196078, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.196078 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7990, + "metrics": { + "recallAt1": 0.019608, + "recallAt5": 0.058824, + "recallAt10": 0.117647, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.355941, + "directRecallAt10": 0, + "supportingRecallAt10": 0.117647 + } + }, + "balancedV1": { + "deliveredTokens": 7987, + "metrics": { + "recallAt1": 0.019608, + "recallAt5": 0.098039, + "recallAt10": 0.156863, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.499433, + "directRecallAt10": 0, + "supportingRecallAt10": 0.156863 + } + }, + "qarinahV2": { + "deliveredTokens": 7990, + "metrics": { + "recallAt1": 0.019608, + "recallAt5": 0.058824, + "recallAt10": 0.117647, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.355941, + "directRecallAt10": 0, + "supportingRecallAt10": 0.117647 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7990, + "metrics": { + "recallAt1": 0.019608, + "recallAt5": 0.058824, + "recallAt10": 0.117647, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.355941, + "directRecallAt10": 0, + "supportingRecallAt10": 0.117647 + } + }, + "oracle": { + "deliveredTokens": 7947, + "metrics": { + "recallAt1": 0.019608, + "recallAt5": 0.098039, + "recallAt10": 0.196078, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.196078 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.5625, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-16873", + "repositorySequence": 111, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 7855, + "balancedV1": 6602, + "qarinahV2": 7855, + "qarinahV2NoGraph": 7855, + "oracle": 268 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 432, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 501, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 432, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 432, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 268, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 889, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 889, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 889, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 268, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1880, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1945, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1880, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1880, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 268, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3992, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 3959, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3992, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3992, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 268, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7990, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 7998, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7990, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7990, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 268, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.439009, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-16910", + "repositorySequence": 112, + "positiveUnderStructuralOracle": true, + "directRecords": 6, + "supportingRecords": 46, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.019231, + "recallAt5": 0.076923, + "recallAt10": 0.173077, + "precisionAt5": 0.8, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.761398, + "directRecallAt10": 0.833333, + "supportingRecallAt10": 0.173077 + }, + "balancedV1": { + "recallAt1": 0.019231, + "recallAt5": 0.096154, + "recallAt10": 0.192308, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.882756, + "directRecallAt10": 0.833333, + "supportingRecallAt10": 0.192308 + }, + "qarinahV2": { + "recallAt1": 0.019231, + "recallAt5": 0.076923, + "recallAt10": 0.173077, + "precisionAt5": 0.8, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.761398, + "directRecallAt10": 0.833333, + "supportingRecallAt10": 0.173077 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.019231, + "recallAt5": 0.076923, + "recallAt10": 0.173077, + "precisionAt5": 0.8, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.761398, + "directRecallAt10": 0.833333, + "supportingRecallAt10": 0.173077 + }, + "oracle": { + "recallAt1": 0.019231, + "recallAt5": 0.096154, + "recallAt10": 0.192308, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.192308 + } + }, + "volumes": { + "bm25Admitted": 5174, + "balancedV1": 5412, + "qarinahV2": 5174, + "qarinahV2NoGraph": 5174, + "oracle": 4388 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.019231, + "recallAt10": 0.019231, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.056571, + "directRecallAt10": 0, + "supportingRecallAt10": 0.019231 + } + }, + "balancedV1": { + "deliveredTokens": 480, + "metrics": { + "recallAt1": 0.019231, + "recallAt5": 0.038462, + "recallAt10": 0.038462, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.325559, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.038462 + } + }, + "qarinahV2": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.019231, + "recallAt10": 0.019231, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.056571, + "directRecallAt10": 0, + "supportingRecallAt10": 0.019231 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.019231, + "recallAt10": 0.019231, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.056571, + "directRecallAt10": 0, + "supportingRecallAt10": 0.019231 + } + }, + "oracle": { + "deliveredTokens": 477, + "metrics": { + "recallAt1": 0.019231, + "recallAt5": 0.038462, + "recallAt10": 0.038462, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.325559, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.038462 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.019231, + "recallAt5": 0.019231, + "recallAt10": 0.019231, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.089663, + "directRecallAt10": 0, + "supportingRecallAt10": 0.019231 + } + }, + "balancedV1": { + "deliveredTokens": 987, + "metrics": { + "recallAt1": 0.019231, + "recallAt5": 0.038462, + "recallAt10": 0.038462, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.438701, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.038462 + } + }, + "qarinahV2": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.019231, + "recallAt5": 0.019231, + "recallAt10": 0.019231, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.089663, + "directRecallAt10": 0, + "supportingRecallAt10": 0.019231 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0.019231, + "recallAt5": 0.019231, + "recallAt10": 0.019231, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.089663, + "directRecallAt10": 0, + "supportingRecallAt10": 0.019231 + } + }, + "oracle": { + "deliveredTokens": 955, + "metrics": { + "recallAt1": 0.019231, + "recallAt5": 0.057692, + "recallAt10": 0.057692, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.573196, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.057692 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1917, + "metrics": { + "recallAt1": 0.019231, + "recallAt5": 0.038462, + "recallAt10": 0.038462, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.259376, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.038462 + } + }, + "balancedV1": { + "deliveredTokens": 1985, + "metrics": { + "recallAt1": 0.019231, + "recallAt5": 0.057692, + "recallAt10": 0.057692, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.460054, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.057692 + } + }, + "qarinahV2": { + "deliveredTokens": 1917, + "metrics": { + "recallAt1": 0.019231, + "recallAt5": 0.038462, + "recallAt10": 0.038462, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.259376, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.038462 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1917, + "metrics": { + "recallAt1": 0.019231, + "recallAt5": 0.038462, + "recallAt10": 0.038462, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.259376, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.038462 + } + }, + "oracle": { + "deliveredTokens": 1966, + "metrics": { + "recallAt1": 0.019231, + "recallAt5": 0.096154, + "recallAt10": 0.096154, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.723729, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.096154 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3959, + "metrics": { + "recallAt1": 0.019231, + "recallAt5": 0.076923, + "recallAt10": 0.134615, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.651918, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.134615 + } + }, + "balancedV1": { + "deliveredTokens": 3985, + "metrics": { + "recallAt1": 0.019231, + "recallAt5": 0.096154, + "recallAt10": 0.153846, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.770071, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.153846 + } + }, + "qarinahV2": { + "deliveredTokens": 3959, + "metrics": { + "recallAt1": 0.019231, + "recallAt5": 0.076923, + "recallAt10": 0.134615, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.651918, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.134615 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3959, + "metrics": { + "recallAt1": 0.019231, + "recallAt5": 0.076923, + "recallAt10": 0.134615, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.651918, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.134615 + } + }, + "oracle": { + "deliveredTokens": 3976, + "metrics": { + "recallAt1": 0.019231, + "recallAt5": 0.096154, + "recallAt10": 0.192308, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.192308 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7969, + "metrics": { + "recallAt1": 0.019231, + "recallAt5": 0.076923, + "recallAt10": 0.173077, + "precisionAt5": 0.8, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.761398, + "directRecallAt10": 0.833333, + "supportingRecallAt10": 0.173077 + } + }, + "balancedV1": { + "deliveredTokens": 7959, + "metrics": { + "recallAt1": 0.019231, + "recallAt5": 0.096154, + "recallAt10": 0.192308, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.882756, + "directRecallAt10": 0.833333, + "supportingRecallAt10": 0.192308 + } + }, + "qarinahV2": { + "deliveredTokens": 7969, + "metrics": { + "recallAt1": 0.019231, + "recallAt5": 0.076923, + "recallAt10": 0.173077, + "precisionAt5": 0.8, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.761398, + "directRecallAt10": 0.833333, + "supportingRecallAt10": 0.173077 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7969, + "metrics": { + "recallAt1": 0.019231, + "recallAt5": 0.076923, + "recallAt10": 0.173077, + "precisionAt5": 0.8, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.761398, + "directRecallAt10": 0.833333, + "supportingRecallAt10": 0.173077 + } + }, + "oracle": { + "deliveredTokens": 7988, + "metrics": { + "recallAt1": 0.019231, + "recallAt5": 0.096154, + "recallAt10": 0.192308, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.192308 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.593966, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-17051", + "repositorySequence": 113, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 52, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.018868, + "recallAt5": 0.075472, + "recallAt10": 0.150943, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.571941, + "directRecallAt10": 0, + "supportingRecallAt10": 0.150943 + }, + "balancedV1": { + "recallAt1": 0.018868, + "recallAt5": 0.075472, + "recallAt10": 0.132075, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.505551, + "directRecallAt10": 0, + "supportingRecallAt10": 0.132075 + }, + "qarinahV2": { + "recallAt1": 0.018868, + "recallAt5": 0.075472, + "recallAt10": 0.150943, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.571941, + "directRecallAt10": 0, + "supportingRecallAt10": 0.150943 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.018868, + "recallAt5": 0.075472, + "recallAt10": 0.150943, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.571941, + "directRecallAt10": 0, + "supportingRecallAt10": 0.150943 + }, + "oracle": { + "recallAt1": 0.018868, + "recallAt5": 0.09434, + "recallAt10": 0.188679, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.188679 + } + }, + "volumes": { + "bm25Admitted": 4661, + "balancedV1": 4899, + "qarinahV2": 4661, + "qarinahV2NoGraph": 4661, + "oracle": 3409 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 471, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 454, + "metrics": { + "recallAt1": 0.018868, + "recallAt5": 0.037736, + "recallAt10": 0.037736, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.037736 + } + }, + "qarinahV2": { + "deliveredTokens": 471, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 471, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 485, + "metrics": { + "recallAt1": 0.018868, + "recallAt5": 0.037736, + "recallAt10": 0.037736, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.037736 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 909, + "metrics": { + "recallAt1": 0.018868, + "recallAt5": 0.018868, + "recallAt10": 0.018868, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.018868 + } + }, + "balancedV1": { + "deliveredTokens": 922, + "metrics": { + "recallAt1": 0.018868, + "recallAt5": 0.037736, + "recallAt10": 0.037736, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.229233, + "directRecallAt10": 0, + "supportingRecallAt10": 0.037736 + } + }, + "qarinahV2": { + "deliveredTokens": 909, + "metrics": { + "recallAt1": 0.018868, + "recallAt5": 0.018868, + "recallAt10": 0.018868, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.018868 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 909, + "metrics": { + "recallAt1": 0.018868, + "recallAt5": 0.018868, + "recallAt10": 0.018868, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.018868 + } + }, + "oracle": { + "deliveredTokens": 962, + "metrics": { + "recallAt1": 0.018868, + "recallAt5": 0.075472, + "recallAt10": 0.075472, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.697114, + "directRecallAt10": 1, + "supportingRecallAt10": 0.075472 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1990, + "metrics": { + "recallAt1": 0.018868, + "recallAt5": 0.056604, + "recallAt10": 0.056604, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.315059, + "directRecallAt10": 0, + "supportingRecallAt10": 0.056604 + } + }, + "balancedV1": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0.018868, + "recallAt5": 0.056604, + "recallAt10": 0.056604, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.29505, + "directRecallAt10": 0, + "supportingRecallAt10": 0.056604 + } + }, + "qarinahV2": { + "deliveredTokens": 1990, + "metrics": { + "recallAt1": 0.018868, + "recallAt5": 0.056604, + "recallAt10": 0.056604, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.315059, + "directRecallAt10": 0, + "supportingRecallAt10": 0.056604 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1990, + "metrics": { + "recallAt1": 0.018868, + "recallAt5": 0.056604, + "recallAt10": 0.056604, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.315059, + "directRecallAt10": 0, + "supportingRecallAt10": 0.056604 + } + }, + "oracle": { + "deliveredTokens": 1953, + "metrics": { + "recallAt1": 0.018868, + "recallAt5": 0.09434, + "recallAt10": 0.113208, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.81067, + "directRecallAt10": 1, + "supportingRecallAt10": 0.113208 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3994, + "metrics": { + "recallAt1": 0.018868, + "recallAt5": 0.075472, + "recallAt10": 0.150943, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.571941, + "directRecallAt10": 0, + "supportingRecallAt10": 0.150943 + } + }, + "balancedV1": { + "deliveredTokens": 3940, + "metrics": { + "recallAt1": 0.018868, + "recallAt5": 0.075472, + "recallAt10": 0.113208, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.459547, + "directRecallAt10": 0, + "supportingRecallAt10": 0.113208 + } + }, + "qarinahV2": { + "deliveredTokens": 3994, + "metrics": { + "recallAt1": 0.018868, + "recallAt5": 0.075472, + "recallAt10": 0.150943, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.571941, + "directRecallAt10": 0, + "supportingRecallAt10": 0.150943 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3994, + "metrics": { + "recallAt1": 0.018868, + "recallAt5": 0.075472, + "recallAt10": 0.150943, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.571941, + "directRecallAt10": 0, + "supportingRecallAt10": 0.150943 + } + }, + "oracle": { + "deliveredTokens": 3977, + "metrics": { + "recallAt1": 0.018868, + "recallAt5": 0.09434, + "recallAt10": 0.188679, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.188679 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7910, + "metrics": { + "recallAt1": 0.018868, + "recallAt5": 0.075472, + "recallAt10": 0.150943, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.571941, + "directRecallAt10": 0, + "supportingRecallAt10": 0.150943 + } + }, + "balancedV1": { + "deliveredTokens": 7877, + "metrics": { + "recallAt1": 0.018868, + "recallAt5": 0.075472, + "recallAt10": 0.132075, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.505551, + "directRecallAt10": 0, + "supportingRecallAt10": 0.132075 + } + }, + "qarinahV2": { + "deliveredTokens": 7910, + "metrics": { + "recallAt1": 0.018868, + "recallAt5": 0.075472, + "recallAt10": 0.150943, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.571941, + "directRecallAt10": 0, + "supportingRecallAt10": 0.150943 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7910, + "metrics": { + "recallAt1": 0.018868, + "recallAt5": 0.075472, + "recallAt10": 0.150943, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.571941, + "directRecallAt10": 0, + "supportingRecallAt10": 0.150943 + } + }, + "oracle": { + "deliveredTokens": 7894, + "metrics": { + "recallAt1": 0.018868, + "recallAt5": 0.09434, + "recallAt10": 0.188679, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.188679 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.454268, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "django/django", + "instanceId": "django__django-17087", + "repositorySequence": 114, + "positiveUnderStructuralOracle": true, + "directRecords": 4, + "supportingRecords": 50, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.018519, + "recallAt5": 0.092593, + "recallAt10": 0.12963, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.525343, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.12963 + }, + "balancedV1": { + "recallAt1": 0.018519, + "recallAt5": 0.092593, + "recallAt10": 0.185185, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.943427, + "directRecallAt10": 1, + "supportingRecallAt10": 0.185185 + }, + "qarinahV2": { + "recallAt1": 0.018519, + "recallAt5": 0.092593, + "recallAt10": 0.12963, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.525343, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.12963 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.018519, + "recallAt5": 0.092593, + "recallAt10": 0.12963, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.525343, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.12963 + }, + "oracle": { + "recallAt1": 0.018519, + "recallAt5": 0.092593, + "recallAt10": 0.185185, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.185185 + } + }, + "volumes": { + "bm25Admitted": 4638, + "balancedV1": 3895, + "qarinahV2": 4638, + "qarinahV2NoGraph": 4638, + "oracle": 3691 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 448, + "metrics": { + "recallAt1": 0.018519, + "recallAt5": 0.018519, + "recallAt10": 0.018519, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.103447, + "directRecallAt10": 0, + "supportingRecallAt10": 0.018519 + } + }, + "balancedV1": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.018519, + "recallAt5": 0.018519, + "recallAt10": 0.018519, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.310341, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.018519 + } + }, + "qarinahV2": { + "deliveredTokens": 448, + "metrics": { + "recallAt1": 0.018519, + "recallAt5": 0.018519, + "recallAt10": 0.018519, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.103447, + "directRecallAt10": 0, + "supportingRecallAt10": 0.018519 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 448, + "metrics": { + "recallAt1": 0.018519, + "recallAt5": 0.018519, + "recallAt10": 0.018519, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.103447, + "directRecallAt10": 0, + "supportingRecallAt10": 0.018519 + } + }, + "oracle": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0.018519, + "recallAt5": 0.018519, + "recallAt10": 0.018519, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.310341, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.018519 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 919, + "metrics": { + "recallAt1": 0.018519, + "recallAt5": 0.055556, + "recallAt10": 0.055556, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.220439, + "directRecallAt10": 0, + "supportingRecallAt10": 0.055556 + } + }, + "balancedV1": { + "deliveredTokens": 914, + "metrics": { + "recallAt1": 0.018519, + "recallAt5": 0.037037, + "recallAt10": 0.037037, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.506145, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.037037 + } + }, + "qarinahV2": { + "deliveredTokens": 919, + "metrics": { + "recallAt1": 0.018519, + "recallAt5": 0.055556, + "recallAt10": 0.055556, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.220439, + "directRecallAt10": 0, + "supportingRecallAt10": 0.055556 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 919, + "metrics": { + "recallAt1": 0.018519, + "recallAt5": 0.055556, + "recallAt10": 0.055556, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.220439, + "directRecallAt10": 0, + "supportingRecallAt10": 0.055556 + } + }, + "oracle": { + "deliveredTokens": 918, + "metrics": { + "recallAt1": 0.018519, + "recallAt5": 0.037037, + "recallAt10": 0.037037, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.506145, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.037037 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1929, + "metrics": { + "recallAt1": 0.018519, + "recallAt5": 0.092593, + "recallAt10": 0.092593, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.30501, + "directRecallAt10": 0, + "supportingRecallAt10": 0.092593 + } + }, + "balancedV1": { + "deliveredTokens": 1987, + "metrics": { + "recallAt1": 0.018519, + "recallAt5": 0.092593, + "recallAt10": 0.092593, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.64244, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.092593 + } + }, + "qarinahV2": { + "deliveredTokens": 1929, + "metrics": { + "recallAt1": 0.018519, + "recallAt5": 0.092593, + "recallAt10": 0.092593, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.30501, + "directRecallAt10": 0, + "supportingRecallAt10": 0.092593 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1929, + "metrics": { + "recallAt1": 0.018519, + "recallAt5": 0.092593, + "recallAt10": 0.092593, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.30501, + "directRecallAt10": 0, + "supportingRecallAt10": 0.092593 + } + }, + "oracle": { + "deliveredTokens": 1987, + "metrics": { + "recallAt1": 0.018519, + "recallAt5": 0.092593, + "recallAt10": 0.111111, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.87184, + "directRecallAt10": 1, + "supportingRecallAt10": 0.111111 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3980, + "metrics": { + "recallAt1": 0.018519, + "recallAt5": 0.092593, + "recallAt10": 0.12963, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.525343, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.12963 + } + }, + "balancedV1": { + "deliveredTokens": 3895, + "metrics": { + "recallAt1": 0.018519, + "recallAt5": 0.092593, + "recallAt10": 0.185185, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.943427, + "directRecallAt10": 1, + "supportingRecallAt10": 0.185185 + } + }, + "qarinahV2": { + "deliveredTokens": 3980, + "metrics": { + "recallAt1": 0.018519, + "recallAt5": 0.092593, + "recallAt10": 0.12963, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.525343, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.12963 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3980, + "metrics": { + "recallAt1": 0.018519, + "recallAt5": 0.092593, + "recallAt10": 0.12963, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.525343, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.12963 + } + }, + "oracle": { + "deliveredTokens": 3986, + "metrics": { + "recallAt1": 0.018519, + "recallAt5": 0.092593, + "recallAt10": 0.185185, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.185185 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7992, + "metrics": { + "recallAt1": 0.018519, + "recallAt5": 0.092593, + "recallAt10": 0.12963, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.525343, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.12963 + } + }, + "balancedV1": { + "deliveredTokens": 7983, + "metrics": { + "recallAt1": 0.018519, + "recallAt5": 0.092593, + "recallAt10": 0.185185, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.943427, + "directRecallAt10": 1, + "supportingRecallAt10": 0.185185 + } + }, + "qarinahV2": { + "deliveredTokens": 7992, + "metrics": { + "recallAt1": 0.018519, + "recallAt5": 0.092593, + "recallAt10": 0.12963, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.525343, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.12963 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7992, + "metrics": { + "recallAt1": 0.018519, + "recallAt5": 0.092593, + "recallAt10": 0.12963, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.525343, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.12963 + } + }, + "oracle": { + "deliveredTokens": 7906, + "metrics": { + "recallAt1": 0.018519, + "recallAt5": 0.092593, + "recallAt10": 0.185185, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.185185 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.491045, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "matplotlib/matplotlib", + "instanceId": "matplotlib__matplotlib-23476", + "repositorySequence": 6, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 4, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2763, + "balancedV1": 2763, + "qarinahV2": 2763, + "qarinahV2NoGraph": 2763, + "oracle": 2470 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 802, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 802, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 802, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1746, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 1944, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2": { + "deliveredTokens": 1746, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1746, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 1961, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2470, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2763, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.95583, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2470, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.46133, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 8, + "noTemporalReturnedItems": 10 + }, + { + "repository": "matplotlib/matplotlib", + "instanceId": "matplotlib__matplotlib-23562", + "repositorySequence": 7, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.356207, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3633, + "balancedV1": 3633, + "qarinahV2": 3633, + "qarinahV2NoGraph": 3633, + "oracle": 293 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 293, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 293, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 293, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 293, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 819, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 819, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 819, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 293, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1689, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1819, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1689, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1689, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 293, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3633, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3633, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.356207, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3633, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3633, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 293, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3633, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3633, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.356207, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3633, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3633, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 293, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.515965, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "matplotlib/matplotlib", + "instanceId": "matplotlib__matplotlib-23563", + "repositorySequence": 8, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.944848, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.504921, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.944848, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.944848, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4005, + "balancedV1": 4005, + "qarinahV2": 4005, + "qarinahV2NoGraph": 4005, + "oracle": 665 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 372, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.826235, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 372, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.826235, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 372, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.826235, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 372, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.826235, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 665, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 665, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 665, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 665, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1535, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.96394, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1898, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.413117, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 1535, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.96394, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1535, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.96394, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 665, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3479, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.944848, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3712, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.413117, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 3479, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.944848, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3479, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.944848, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 665, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 4005, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.944848, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 4005, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.504921, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 4005, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.944848, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 4005, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.944848, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 665, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.454063, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "matplotlib/matplotlib", + "instanceId": "matplotlib__matplotlib-23913", + "repositorySequence": 9, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 5, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.747705, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.746251, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.747705, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.747705, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 5225, + "balancedV1": 5225, + "qarinahV2": 5225, + "qarinahV2NoGraph": 5225, + "oracle": 4560 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.188513, + "directRecallAt10": 0, + "supportingRecallAt10": 0.166667 + } + }, + "balancedV1": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.188513, + "directRecallAt10": 0, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.188513, + "directRecallAt10": 0, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.188513, + "directRecallAt10": 0, + "supportingRecallAt10": 0.166667 + } + }, + "oracle": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.188513, + "directRecallAt10": 0, + "supportingRecallAt10": 0.166667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 784, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.188513, + "directRecallAt10": 0, + "supportingRecallAt10": 0.166667 + } + }, + "balancedV1": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.307452, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 784, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.188513, + "directRecallAt10": 0, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 784, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.188513, + "directRecallAt10": 0, + "supportingRecallAt10": 0.166667 + } + }, + "oracle": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.188513, + "directRecallAt10": 0, + "supportingRecallAt10": 0.166667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1819, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.363958, + "directRecallAt10": 0, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 1898, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.38864, + "directRecallAt10": 0, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 1819, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.363958, + "directRecallAt10": 0, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1819, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.363958, + "directRecallAt10": 0, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 1711, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.684478, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3411, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.625398, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 3411, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.623944, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3411, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.625398, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3411, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.625398, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 3690, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 0.833333, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.93285, + "directRecallAt10": 1, + "supportingRecallAt10": 0.833333 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 5225, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.747705, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 5225, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.746251, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 5225, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.747705, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 5225, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.747705, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 4560, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.425893, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "SINGLE_OR_NO_SUPPORT", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "matplotlib/matplotlib", + "instanceId": "matplotlib__matplotlib-23964", + "repositorySequence": 10, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 6, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.725728, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.702805, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.725728, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.725728, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 5439, + "balancedV1": 5439, + "qarinahV2": 5439, + "qarinahV2NoGraph": 5439, + "oracle": 3554 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 372, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 372, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 372, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 372, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 214, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.302602, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 870, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.302602, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "balancedV1": { + "deliveredTokens": 881, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.190921, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2": { + "deliveredTokens": 870, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.302602, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 870, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.302602, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "oracle": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.302602, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1885, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1806, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.151301, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2": { + "deliveredTokens": 1885, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1885, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 1649, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.644824, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3915, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.438307, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 3969, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.666667, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.506477, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3915, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.438307, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3915, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.438307, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 3554, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 5439, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.725728, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 5439, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.702805, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 5439, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.725728, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 5439, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 0.5, + "ndcgAt10": 0.725728, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 3554, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.538286, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "matplotlib/matplotlib", + "instanceId": "matplotlib__matplotlib-23987", + "repositorySequence": 11, + "positiveUnderStructuralOracle": true, + "directRecords": 3, + "supportingRecords": 5, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.708909, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.125, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.743551, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.708909, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.708909, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 6590, + "balancedV1": 6590, + "qarinahV2": 6590, + "qarinahV2NoGraph": 6590, + "oracle": 5925 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 507, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.230397, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.125 + } + }, + "balancedV1": { + "deliveredTokens": 509, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.121724, + "directRecallAt10": 0, + "supportingRecallAt10": 0.125 + } + }, + "qarinahV2": { + "deliveredTokens": 507, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.230397, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.125 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 507, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.230397, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.125 + } + }, + "oracle": { + "deliveredTokens": 214, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.365171, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.125 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 802, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.076799, + "directRecallAt10": 0, + "supportingRecallAt10": 0.125 + } + }, + "balancedV1": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.198523, + "directRecallAt10": 0, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2": { + "deliveredTokens": 802, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.076799, + "directRecallAt10": 0, + "supportingRecallAt10": 0.125 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 802, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.076799, + "directRecallAt10": 0, + "supportingRecallAt10": 0.125 + } + }, + "oracle": { + "deliveredTokens": 705, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.44197, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.25 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1886, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.375, + "recallAt10": 0.375, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.44853, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.375 + } + }, + "balancedV1": { + "deliveredTokens": 1879, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.375, + "recallAt10": 0.375, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.33979, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.375 + } + }, + "qarinahV2": { + "deliveredTokens": 1886, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.375, + "recallAt10": 0.375, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.44853, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.375 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1886, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.375, + "recallAt10": 0.375, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.44853, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.375 + } + }, + "oracle": { + "deliveredTokens": 1925, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.375, + "recallAt10": 0.375, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.656431, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.375 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3836, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.48495, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 3969, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.5, + "recallAt10": 0.625, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.6441, + "directRecallAt10": 1, + "supportingRecallAt10": 0.625 + } + }, + "qarinahV2": { + "deliveredTokens": 3836, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.48495, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3836, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.48495, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 3739, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 0.625, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.877667, + "directRecallAt10": 1, + "supportingRecallAt10": 0.625 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 6590, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.708909, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 6590, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.743551, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 6590, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.708909, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 6590, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.708909, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 5925, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.598701, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "matplotlib/matplotlib", + "instanceId": "matplotlib__matplotlib-24149", + "repositorySequence": 12, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 8, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.125, + "recallAt5": 0.5, + "recallAt10": 0.875, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.866851, + "directRecallAt10": null, + "supportingRecallAt10": 0.875 + }, + "balancedV1": { + "recallAt1": 0.125, + "recallAt5": 0.375, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.924937, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.125, + "recallAt5": 0.5, + "recallAt10": 0.875, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.866851, + "directRecallAt10": null, + "supportingRecallAt10": 0.875 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.125, + "recallAt5": 0.5, + "recallAt10": 0.875, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.866851, + "directRecallAt10": null, + "supportingRecallAt10": 0.875 + }, + "oracle": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 6753, + "balancedV1": 6674, + "qarinahV2": 6753, + "qarinahV2NoGraph": 6753, + "oracle": 5082 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.252943, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + }, + "balancedV1": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.252943, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + }, + "qarinahV2": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.252943, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.252943, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + }, + "oracle": { + "deliveredTokens": 377, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.252943, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.252943, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + }, + "balancedV1": { + "deliveredTokens": 870, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.252943, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + }, + "qarinahV2": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.252943, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.252943, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + }, + "oracle": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.252943, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1935, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.412532, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 1870, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.375, + "recallAt10": 0.375, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.539003, + "directRecallAt10": null, + "supportingRecallAt10": 0.375 + } + }, + "qarinahV2": { + "deliveredTokens": 1935, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.412532, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1935, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.412532, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 1535, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.375, + "recallAt10": 0.375, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.539003, + "directRecallAt10": null, + "supportingRecallAt10": 0.375 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3806, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.375, + "recallAt10": 0.375, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.521468, + "directRecallAt10": null, + "supportingRecallAt10": 0.375 + } + }, + "balancedV1": { + "deliveredTokens": 3839, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.375, + "recallAt10": 0.5, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.611568, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 3806, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.375, + "recallAt10": 0.375, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.521468, + "directRecallAt10": null, + "supportingRecallAt10": 0.375 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3806, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.375, + "recallAt10": 0.375, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.521468, + "directRecallAt10": null, + "supportingRecallAt10": 0.375 + } + }, + "oracle": { + "deliveredTokens": 3703, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 0.75, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.835891, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 6967, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.5, + "recallAt10": 0.875, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.866851, + "directRecallAt10": null, + "supportingRecallAt10": 0.875 + } + }, + "balancedV1": { + "deliveredTokens": 6967, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.375, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.924937, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 6967, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.5, + "recallAt10": 0.875, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.866851, + "directRecallAt10": null, + "supportingRecallAt10": 0.875 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 6967, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.5, + "recallAt10": 0.875, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.866851, + "directRecallAt10": null, + "supportingRecallAt10": 0.875 + } + }, + "oracle": { + "deliveredTokens": 5082, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.476923, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "matplotlib/matplotlib", + "instanceId": "matplotlib__matplotlib-24265", + "repositorySequence": 13, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 8, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.111111, + "recallAt5": 0.444444, + "recallAt10": 0.777778, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.520488, + "directRecallAt10": 0, + "supportingRecallAt10": 0.777778 + }, + "balancedV1": { + "recallAt1": 0.111111, + "recallAt5": 0.333333, + "recallAt10": 0.777778, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.650427, + "directRecallAt10": 1, + "supportingRecallAt10": 0.777778 + }, + "qarinahV2": { + "recallAt1": 0.111111, + "recallAt5": 0.444444, + "recallAt10": 0.777778, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.520488, + "directRecallAt10": 0, + "supportingRecallAt10": 0.777778 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.111111, + "recallAt5": 0.444444, + "recallAt10": 0.777778, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.520488, + "directRecallAt10": 0, + "supportingRecallAt10": 0.777778 + }, + "oracle": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 7125, + "balancedV1": 7090, + "qarinahV2": 7125, + "qarinahV2NoGraph": 7125, + "oracle": 5945 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 377, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.159885, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + } + }, + "balancedV1": { + "deliveredTokens": 377, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.159885, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + } + }, + "qarinahV2": { + "deliveredTokens": 377, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.159885, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 377, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.159885, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + } + }, + "oracle": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.479655, + "directRecallAt10": 1, + "supportingRecallAt10": 0.111111 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 884, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.239828, + "directRecallAt10": 0, + "supportingRecallAt10": 0.222222 + } + }, + "balancedV1": { + "deliveredTokens": 886, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.260761, + "directRecallAt10": 0, + "supportingRecallAt10": 0.222222 + } + }, + "qarinahV2": { + "deliveredTokens": 884, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.239828, + "directRecallAt10": 0, + "supportingRecallAt10": 0.222222 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 884, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.239828, + "directRecallAt10": 0, + "supportingRecallAt10": 0.222222 + } + }, + "oracle": { + "deliveredTokens": 868, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.580531, + "directRecallAt10": 1, + "supportingRecallAt10": 0.222222 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1890, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.159885, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + } + }, + "balancedV1": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.444444, + "recallAt10": 0.444444, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.409562, + "directRecallAt10": 0, + "supportingRecallAt10": 0.444444 + } + }, + "qarinahV2": { + "deliveredTokens": 1890, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.159885, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1890, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.159885, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + } + }, + "oracle": { + "deliveredTokens": 1812, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.660474, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3995, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.308686, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 3839, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.333333, + "recallAt10": 0.444444, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.500477, + "directRecallAt10": 1, + "supportingRecallAt10": 0.444444 + } + }, + "qarinahV2": { + "deliveredTokens": 3995, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.308686, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3995, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.308686, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 3703, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 0.666667, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.848137, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7830, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.444444, + "recallAt10": 0.777778, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.520488, + "directRecallAt10": 0, + "supportingRecallAt10": 0.777778 + } + }, + "balancedV1": { + "deliveredTokens": 7830, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.333333, + "recallAt10": 0.777778, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.650427, + "directRecallAt10": 1, + "supportingRecallAt10": 0.777778 + } + }, + "qarinahV2": { + "deliveredTokens": 7830, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.444444, + "recallAt10": 0.777778, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.520488, + "directRecallAt10": 0, + "supportingRecallAt10": 0.777778 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7830, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.444444, + "recallAt10": 0.777778, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.520488, + "directRecallAt10": 0, + "supportingRecallAt10": 0.777778 + } + }, + "oracle": { + "deliveredTokens": 5945, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.615217, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "matplotlib/matplotlib", + "instanceId": "matplotlib__matplotlib-24334", + "repositorySequence": 14, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 10, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 0.8, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.857981, + "directRecallAt10": null, + "supportingRecallAt10": 0.8 + }, + "balancedV1": { + "recallAt1": 0.1, + "recallAt5": 0.3, + "recallAt10": 0.8, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.795166, + "directRecallAt10": null, + "supportingRecallAt10": 0.8 + }, + "qarinahV2": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 0.8, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.857981, + "directRecallAt10": null, + "supportingRecallAt10": 0.8 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 0.8, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.857981, + "directRecallAt10": null, + "supportingRecallAt10": 0.8 + }, + "oracle": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 6038, + "balancedV1": 6593, + "qarinahV2": 6038, + "qarinahV2NoGraph": 6038, + "oracle": 6339 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.1 + } + }, + "balancedV1": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.1 + } + }, + "qarinahV2": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.1 + } + }, + "oracle": { + "deliveredTokens": 394, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.1, + "recallAt10": 0.1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "balancedV1": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "oracle": { + "deliveredTokens": 985, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.3, + "recallAt10": 0.3, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.3 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1880, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.3, + "recallAt10": 0.3, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.3 + } + }, + "balancedV1": { + "deliveredTokens": 1963, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.538886, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2": { + "deliveredTokens": 1880, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.3, + "recallAt10": 0.3, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.3 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1880, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.3, + "recallAt10": 0.3, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.3 + } + }, + "oracle": { + "deliveredTokens": 1929, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3934, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 0.6, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.6 + } + }, + "balancedV1": { + "deliveredTokens": 3839, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.3, + "recallAt10": 0.4, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.522496, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2": { + "deliveredTokens": 3934, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 0.6, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.6 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3934, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 0.6, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.6 + } + }, + "oracle": { + "deliveredTokens": 3571, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 0.6, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.6 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7354, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 0.8, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.857981, + "directRecallAt10": null, + "supportingRecallAt10": 0.8 + } + }, + "balancedV1": { + "deliveredTokens": 7931, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.3, + "recallAt10": 0.8, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.795166, + "directRecallAt10": null, + "supportingRecallAt10": 0.8 + } + }, + "qarinahV2": { + "deliveredTokens": 7354, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 0.8, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.857981, + "directRecallAt10": null, + "supportingRecallAt10": 0.8 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7354, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 0.8, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.857981, + "directRecallAt10": null, + "supportingRecallAt10": 0.8 + } + }, + "oracle": { + "deliveredTokens": 6339, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.423182, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "matplotlib/matplotlib", + "instanceId": "matplotlib__matplotlib-24970", + "repositorySequence": 15, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.787774, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.70151, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.787774, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.787774, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 7147, + "balancedV1": 7191, + "qarinahV2": 7147, + "qarinahV2NoGraph": 7147, + "oracle": 6069 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 394, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 377, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2": { + "deliveredTokens": 394, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 394, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 394, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 868, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 847, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1807, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 1969, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2": { + "deliveredTokens": 1807, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1807, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 1954, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3928, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.363636, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.503324, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "balancedV1": { + "deliveredTokens": 3839, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.42884, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2": { + "deliveredTokens": 3928, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.363636, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.503324, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3928, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.363636, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.503324, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "oracle": { + "deliveredTokens": 3810, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.636364, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.800694, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7972, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.787774, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "balancedV1": { + "deliveredTokens": 7937, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.70151, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2": { + "deliveredTokens": 7972, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.787774, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7972, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.727273, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.787774, + "directRecallAt10": null, + "supportingRecallAt10": 0.727273 + } + }, + "oracle": { + "deliveredTokens": 6578, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.587975, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "matplotlib/matplotlib", + "instanceId": "matplotlib__matplotlib-25079", + "repositorySequence": 16, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.497358, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.666667, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.475717, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.497358, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.497358, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + }, + "oracle": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.833333, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.833333 + } + }, + "volumes": { + "bm25Admitted": 7226, + "balancedV1": 7323, + "qarinahV2": 7226, + "qarinahV2NoGraph": 7226, + "oracle": 5689 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 377, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.083333, + "recallAt10": 0.083333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.083333 + } + }, + "balancedV1": { + "deliveredTokens": 377, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.083333, + "recallAt10": 0.083333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.083333 + } + }, + "qarinahV2": { + "deliveredTokens": 377, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.083333, + "recallAt10": 0.083333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.083333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 377, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.083333, + "recallAt10": 0.083333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.083333 + } + }, + "oracle": { + "deliveredTokens": 490, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.083333, + "recallAt10": 0.083333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.458466, + "directRecallAt10": 1, + "supportingRecallAt10": 0.083333 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.083333, + "recallAt10": 0.083333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.083333 + } + }, + "balancedV1": { + "deliveredTokens": 988, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.229233, + "directRecallAt10": 0, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.083333, + "recallAt10": 0.083333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.083333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.083333, + "recallAt10": 0.083333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.083333 + } + }, + "oracle": { + "deliveredTokens": 884, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.166667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.172831, + "directRecallAt10": 0, + "supportingRecallAt10": 0.166667 + } + }, + "balancedV1": { + "deliveredTokens": 1969, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.083333, + "recallAt10": 0.083333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.09642, + "directRecallAt10": 0, + "supportingRecallAt10": 0.083333 + } + }, + "qarinahV2": { + "deliveredTokens": 1991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.172831, + "directRecallAt10": 0, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.172831, + "directRecallAt10": 0, + "supportingRecallAt10": 0.166667 + } + }, + "oracle": { + "deliveredTokens": 1828, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.631297, + "directRecallAt10": 1, + "supportingRecallAt10": 0.25 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3985, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.238648, + "directRecallAt10": 0, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 3839, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.333333, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.286387, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 3985, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.238648, + "directRecallAt10": 0, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3985, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.238648, + "directRecallAt10": 0, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 3675, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.666667, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.909821, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7955, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.497358, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 7956, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.666667, + "precisionAt5": 0.6, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.475717, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 7955, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.497358, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7955, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 0.5, + "ndcgAt10": 0.497358, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 7068, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.833333, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.833333 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.578671, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "matplotlib/matplotlib", + "instanceId": "matplotlib__matplotlib-25311", + "repositorySequence": 17, + "positiveUnderStructuralOracle": true, + "directRecords": 3, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.642857, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.764849, + "directRecallAt10": 1, + "supportingRecallAt10": 0.642857 + }, + "balancedV1": { + "recallAt1": 0.071429, + "recallAt5": 0.285714, + "recallAt10": 0.642857, + "precisionAt5": 0.8, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.778844, + "directRecallAt10": 1, + "supportingRecallAt10": 0.642857 + }, + "qarinahV2": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.642857, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.764849, + "directRecallAt10": 1, + "supportingRecallAt10": 0.642857 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.642857, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.764849, + "directRecallAt10": 1, + "supportingRecallAt10": 0.642857 + }, + "oracle": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.714286, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.714286 + } + }, + "volumes": { + "bm25Admitted": 7047, + "balancedV1": 6690, + "qarinahV2": 7047, + "qarinahV2NoGraph": 7047, + "oracle": 6889 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 507, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.340699, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.071429 + } + }, + "balancedV1": { + "deliveredTokens": 377, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.340699, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.071429 + } + }, + "qarinahV2": { + "deliveredTokens": 507, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.340699, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.071429 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 507, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.340699, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.071429 + } + }, + "oracle": { + "deliveredTokens": 377, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.340699, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.071429 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 870, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.113566, + "directRecallAt10": 0, + "supportingRecallAt10": 0.071429 + } + }, + "balancedV1": { + "deliveredTokens": 870, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.113566, + "directRecallAt10": 0, + "supportingRecallAt10": 0.071429 + } + }, + "qarinahV2": { + "deliveredTokens": 870, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.113566, + "directRecallAt10": 0, + "supportingRecallAt10": 0.071429 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 870, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.113566, + "directRecallAt10": 0, + "supportingRecallAt10": 0.071429 + } + }, + "oracle": { + "deliveredTokens": 985, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.214286, + "recallAt10": 0.214286, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.61244, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.214286 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1998, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.434217, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.285714 + } + }, + "balancedV1": { + "deliveredTokens": 1833, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.214286, + "recallAt10": 0.214286, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.475255, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.214286 + } + }, + "qarinahV2": { + "deliveredTokens": 1998, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.434217, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.285714 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1998, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.434217, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.285714 + } + }, + "oracle": { + "deliveredTokens": 1811, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.214286, + "recallAt10": 0.214286, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.726006, + "directRecallAt10": 1, + "supportingRecallAt10": 0.214286 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3849, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.357143, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.546233, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.357143 + } + }, + "balancedV1": { + "deliveredTokens": 3835, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.532038, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.285714 + } + }, + "qarinahV2": { + "deliveredTokens": 3849, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.357143, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.546233, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.357143 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3849, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.357143, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.546233, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.357143 + } + }, + "oracle": { + "deliveredTokens": 3879, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.5, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.897159, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7910, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.642857, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.764849, + "directRecallAt10": 1, + "supportingRecallAt10": 0.642857 + } + }, + "balancedV1": { + "deliveredTokens": 7903, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.285714, + "recallAt10": 0.642857, + "precisionAt5": 0.8, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.778844, + "directRecallAt10": 1, + "supportingRecallAt10": 0.642857 + } + }, + "qarinahV2": { + "deliveredTokens": 7910, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.642857, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.764849, + "directRecallAt10": 1, + "supportingRecallAt10": 0.642857 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7910, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.642857, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.764849, + "directRecallAt10": 1, + "supportingRecallAt10": 0.642857 + } + }, + "oracle": { + "deliveredTokens": 7905, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.714286, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.714286 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.602381, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "matplotlib/matplotlib", + "instanceId": "matplotlib__matplotlib-25332", + "repositorySequence": 18, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 13, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.071429, + "recallAt5": 0.285714, + "recallAt10": 0.571429, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.77364, + "directRecallAt10": 1, + "supportingRecallAt10": 0.571429 + }, + "balancedV1": { + "recallAt1": 0.071429, + "recallAt5": 0.285714, + "recallAt10": 0.571429, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.770438, + "directRecallAt10": 1, + "supportingRecallAt10": 0.571429 + }, + "qarinahV2": { + "recallAt1": 0.071429, + "recallAt5": 0.285714, + "recallAt10": 0.571429, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.77364, + "directRecallAt10": 1, + "supportingRecallAt10": 0.571429 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.071429, + "recallAt5": 0.285714, + "recallAt10": 0.571429, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.77364, + "directRecallAt10": 1, + "supportingRecallAt10": 0.571429 + }, + "oracle": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.714286, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.714286 + } + }, + "volumes": { + "bm25Admitted": 7507, + "balancedV1": 6760, + "qarinahV2": 7507, + "qarinahV2NoGraph": 7507, + "oracle": 5960 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 498, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + } + }, + "balancedV1": { + "deliveredTokens": 377, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.071429 + } + }, + "qarinahV2": { + "deliveredTokens": 498, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 498, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + } + }, + "oracle": { + "deliveredTokens": 394, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.071429 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 793, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + } + }, + "balancedV1": { + "deliveredTokens": 886, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2": { + "deliveredTokens": 793, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 793, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + } + }, + "oracle": { + "deliveredTokens": 870, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.458466, + "directRecallAt10": 1, + "supportingRecallAt10": 0.071429 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1956, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.214286, + "recallAt10": 0.214286, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.518493, + "directRecallAt10": 1, + "supportingRecallAt10": 0.214286 + } + }, + "balancedV1": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.58431, + "directRecallAt10": 1, + "supportingRecallAt10": 0.285714 + } + }, + "qarinahV2": { + "deliveredTokens": 1956, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.214286, + "recallAt10": 0.214286, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.518493, + "directRecallAt10": 1, + "supportingRecallAt10": 0.214286 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1956, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.214286, + "recallAt10": 0.214286, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.518493, + "directRecallAt10": 1, + "supportingRecallAt10": 0.214286 + } + }, + "oracle": { + "deliveredTokens": 1787, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.697114, + "directRecallAt10": 1, + "supportingRecallAt10": 0.285714 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3878, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.285714, + "recallAt10": 0.428571, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.689687, + "directRecallAt10": 1, + "supportingRecallAt10": 0.428571 + } + }, + "balancedV1": { + "deliveredTokens": 3972, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.577613, + "directRecallAt10": 1, + "supportingRecallAt10": 0.285714 + } + }, + "qarinahV2": { + "deliveredTokens": 3878, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.285714, + "recallAt10": 0.428571, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.689687, + "directRecallAt10": 1, + "supportingRecallAt10": 0.428571 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3878, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.285714, + "recallAt10": 0.428571, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.689687, + "directRecallAt10": 1, + "supportingRecallAt10": 0.428571 + } + }, + "oracle": { + "deliveredTokens": 3941, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.5, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.861611, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7901, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.285714, + "recallAt10": 0.571429, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.77364, + "directRecallAt10": 1, + "supportingRecallAt10": 0.571429 + } + }, + "balancedV1": { + "deliveredTokens": 7916, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.285714, + "recallAt10": 0.571429, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.770438, + "directRecallAt10": 1, + "supportingRecallAt10": 0.571429 + } + }, + "qarinahV2": { + "deliveredTokens": 7901, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.285714, + "recallAt10": 0.571429, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.77364, + "directRecallAt10": 1, + "supportingRecallAt10": 0.571429 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7901, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.285714, + "recallAt10": 0.571429, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.77364, + "directRecallAt10": 1, + "supportingRecallAt10": 0.571429 + } + }, + "oracle": { + "deliveredTokens": 7839, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.714286, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.714286 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "DIRECTLY_SUPPORTED", + "decision": "ACCEPT_DIRECT", + "score": 0.736829, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "CONSERVATIVE_DIRECT_THRESHOLD_MET" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "matplotlib/matplotlib", + "instanceId": "matplotlib__matplotlib-25433", + "repositorySequence": 19, + "positiveUnderStructuralOracle": true, + "directRecords": 2, + "supportingRecords": 13, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.066667, + "recallAt5": 0.266667, + "recallAt10": 0.6, + "precisionAt5": 0.8, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.751929, + "directRecallAt10": 1, + "supportingRecallAt10": 0.6 + }, + "balancedV1": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.533333, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.770999, + "directRecallAt10": 1, + "supportingRecallAt10": 0.533333 + }, + "qarinahV2": { + "recallAt1": 0.066667, + "recallAt5": 0.266667, + "recallAt10": 0.6, + "precisionAt5": 0.8, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.751929, + "directRecallAt10": 1, + "supportingRecallAt10": 0.6 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.066667, + "recallAt5": 0.266667, + "recallAt10": 0.6, + "precisionAt5": 0.8, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.751929, + "directRecallAt10": 1, + "supportingRecallAt10": 0.6 + }, + "oracle": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "volumes": { + "bm25Admitted": 7705, + "balancedV1": 6553, + "qarinahV2": 7705, + "qarinahV2NoGraph": 7705, + "oracle": 5123 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 498, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.133333, + "recallAt10": 0.133333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.208948, + "directRecallAt10": 0, + "supportingRecallAt10": 0.133333 + } + }, + "balancedV1": { + "deliveredTokens": 490, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.066667, + "recallAt10": 0.066667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.128116, + "directRecallAt10": 0, + "supportingRecallAt10": 0.066667 + } + }, + "qarinahV2": { + "deliveredTokens": 498, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.133333, + "recallAt10": 0.133333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.208948, + "directRecallAt10": 0, + "supportingRecallAt10": 0.133333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 498, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.133333, + "recallAt10": 0.133333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.208948, + "directRecallAt10": 0, + "supportingRecallAt10": 0.133333 + } + }, + "oracle": { + "deliveredTokens": 377, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.066667, + "recallAt10": 0.066667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.384348, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.066667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.066667, + "recallAt10": 0.066667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.128116, + "directRecallAt10": 0, + "supportingRecallAt10": 0.066667 + } + }, + "balancedV1": { + "deliveredTokens": 996, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.066667, + "recallAt10": 0.066667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.128116, + "directRecallAt10": 0, + "supportingRecallAt10": 0.066667 + } + }, + "qarinahV2": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.066667, + "recallAt10": 0.066667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.128116, + "directRecallAt10": 0, + "supportingRecallAt10": 0.066667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.066667, + "recallAt10": 0.066667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.128116, + "directRecallAt10": 0, + "supportingRecallAt10": 0.066667 + } + }, + "oracle": { + "deliveredTokens": 985, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.529239, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.2 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1940, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.133333, + "recallAt10": 0.133333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.208948, + "directRecallAt10": 0, + "supportingRecallAt10": 0.133333 + } + }, + "balancedV1": { + "deliveredTokens": 1866, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.133333, + "recallAt10": 0.133333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.370613, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.133333 + } + }, + "qarinahV2": { + "deliveredTokens": 1940, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.133333, + "recallAt10": 0.133333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.208948, + "directRecallAt10": 0, + "supportingRecallAt10": 0.133333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1940, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.133333, + "recallAt10": 0.133333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.208948, + "directRecallAt10": 0, + "supportingRecallAt10": 0.133333 + } + }, + "oracle": { + "deliveredTokens": 1925, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.266667, + "recallAt10": 0.266667, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.74608, + "directRecallAt10": 1, + "supportingRecallAt10": 0.266667 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3962, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.4, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.650621, + "directRecallAt10": 1, + "supportingRecallAt10": 0.4 + } + }, + "balancedV1": { + "deliveredTokens": 3972, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.466667, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.738104, + "directRecallAt10": 1, + "supportingRecallAt10": 0.466667 + } + }, + "qarinahV2": { + "deliveredTokens": 3962, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.4, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.650621, + "directRecallAt10": 1, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3962, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.4, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.650621, + "directRecallAt10": 1, + "supportingRecallAt10": 0.4 + } + }, + "oracle": { + "deliveredTokens": 3636, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.533333, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.924399, + "directRecallAt10": 1, + "supportingRecallAt10": 0.533333 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7944, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.266667, + "recallAt10": 0.6, + "precisionAt5": 0.8, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.751929, + "directRecallAt10": 1, + "supportingRecallAt10": 0.6 + } + }, + "balancedV1": { + "deliveredTokens": 7810, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.533333, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.770999, + "directRecallAt10": 1, + "supportingRecallAt10": 0.533333 + } + }, + "qarinahV2": { + "deliveredTokens": 7944, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.266667, + "recallAt10": 0.6, + "precisionAt5": 0.8, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.751929, + "directRecallAt10": 1, + "supportingRecallAt10": 0.6 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7944, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.266667, + "recallAt10": 0.6, + "precisionAt5": 0.8, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.751929, + "directRecallAt10": 1, + "supportingRecallAt10": 0.6 + } + }, + "oracle": { + "deliveredTokens": 7627, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.614205, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "matplotlib/matplotlib", + "instanceId": "matplotlib__matplotlib-25442", + "repositorySequence": 20, + "positiveUnderStructuralOracle": true, + "directRecords": 4, + "supportingRecords": 13, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.058824, + "recallAt5": 0.235294, + "recallAt10": 0.529412, + "precisionAt5": 0.8, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.595728, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.529412 + }, + "balancedV1": { + "recallAt1": 0.058824, + "recallAt5": 0.235294, + "recallAt10": 0.529412, + "precisionAt5": 0.8, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.705859, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.529412 + }, + "qarinahV2": { + "recallAt1": 0.058824, + "recallAt5": 0.235294, + "recallAt10": 0.529412, + "precisionAt5": 0.8, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.595728, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.529412 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.058824, + "recallAt5": 0.235294, + "recallAt10": 0.529412, + "precisionAt5": 0.8, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.595728, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.529412 + }, + "oracle": { + "recallAt1": 0.058824, + "recallAt5": 0.294118, + "recallAt10": 0.588235, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.588235 + } + }, + "volumes": { + "bm25Admitted": 7090, + "balancedV1": 6699, + "qarinahV2": 7090, + "qarinahV2NoGraph": 7090, + "oracle": 5493 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 372, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 511, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.058824, + "recallAt10": 0.058824, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.103447, + "directRecallAt10": 0, + "supportingRecallAt10": 0.058824 + } + }, + "qarinahV2": { + "deliveredTokens": 372, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 372, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 498, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.117647, + "recallAt10": 0.117647, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.506145, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.117647 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 863, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.058824, + "recallAt10": 0.058824, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.103447, + "directRecallAt10": 0, + "supportingRecallAt10": 0.058824 + } + }, + "balancedV1": { + "deliveredTokens": 883, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.058824, + "recallAt10": 0.058824, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.103447, + "directRecallAt10": 0, + "supportingRecallAt10": 0.058824 + } + }, + "qarinahV2": { + "deliveredTokens": 863, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.058824, + "recallAt10": 0.058824, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.103447, + "directRecallAt10": 0, + "supportingRecallAt10": 0.058824 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 863, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.058824, + "recallAt10": 0.058824, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.103447, + "directRecallAt10": 0, + "supportingRecallAt10": 0.058824 + } + }, + "oracle": { + "deliveredTokens": 875, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.176471, + "recallAt10": 0.176471, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.661316, + "directRecallAt10": 0.75, + "supportingRecallAt10": 0.176471 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1972, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.176471, + "recallAt10": 0.176471, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.220439, + "directRecallAt10": 0, + "supportingRecallAt10": 0.176471 + } + }, + "balancedV1": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.176471, + "recallAt10": 0.176471, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.427333, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.176471 + } + }, + "qarinahV2": { + "deliveredTokens": 1972, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.176471, + "recallAt10": 0.176471, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.220439, + "directRecallAt10": 0, + "supportingRecallAt10": 0.176471 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1972, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.176471, + "recallAt10": 0.176471, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.220439, + "directRecallAt10": 0, + "supportingRecallAt10": 0.176471 + } + }, + "oracle": { + "deliveredTokens": 1780, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.294118, + "recallAt10": 0.294118, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.745887, + "directRecallAt10": 0.75, + "supportingRecallAt10": 0.294118 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3890, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.235294, + "recallAt10": 0.235294, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.443942, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.235294 + } + }, + "balancedV1": { + "deliveredTokens": 3969, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.235294, + "recallAt10": 0.235294, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.471885, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.235294 + } + }, + "qarinahV2": { + "deliveredTokens": 3890, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.235294, + "recallAt10": 0.235294, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.443942, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.235294 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3890, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.235294, + "recallAt10": 0.235294, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.443942, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.235294 + } + }, + "oracle": { + "deliveredTokens": 3944, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.294118, + "recallAt10": 0.411765, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.906322, + "directRecallAt10": 1, + "supportingRecallAt10": 0.411765 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7861, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.235294, + "recallAt10": 0.529412, + "precisionAt5": 0.8, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.595728, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.529412 + } + }, + "balancedV1": { + "deliveredTokens": 7850, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.235294, + "recallAt10": 0.529412, + "precisionAt5": 0.8, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.705859, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.529412 + } + }, + "qarinahV2": { + "deliveredTokens": 7861, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.235294, + "recallAt10": 0.529412, + "precisionAt5": 0.8, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.595728, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.529412 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7861, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.235294, + "recallAt10": 0.529412, + "precisionAt5": 0.8, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.595728, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.529412 + } + }, + "oracle": { + "deliveredTokens": 7998, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.294118, + "recallAt10": 0.588235, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.588235 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.605293, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "matplotlib/matplotlib", + "instanceId": "matplotlib__matplotlib-25498", + "repositorySequence": 21, + "positiveUnderStructuralOracle": true, + "directRecords": 5, + "supportingRecords": 13, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.055556, + "recallAt5": 0.222222, + "recallAt10": 0.5, + "precisionAt5": 0.8, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.554346, + "directRecallAt10": 0.4, + "supportingRecallAt10": 0.5 + }, + "balancedV1": { + "recallAt1": 0.055556, + "recallAt5": 0.222222, + "recallAt10": 0.5, + "precisionAt5": 0.8, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.658001, + "directRecallAt10": 0.8, + "supportingRecallAt10": 0.5 + }, + "qarinahV2": { + "recallAt1": 0.055556, + "recallAt5": 0.222222, + "recallAt10": 0.5, + "precisionAt5": 0.8, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.554346, + "directRecallAt10": 0.4, + "supportingRecallAt10": 0.5 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.055556, + "recallAt5": 0.222222, + "recallAt10": 0.5, + "precisionAt5": 0.8, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.554346, + "directRecallAt10": 0.4, + "supportingRecallAt10": 0.5 + }, + "oracle": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.555556, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.555556 + } + }, + "volumes": { + "bm25Admitted": 7977, + "balancedV1": 6154, + "qarinahV2": 7977, + "qarinahV2NoGraph": 7977, + "oracle": 5799 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 372, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 372, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 372, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 372, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 498, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.468637, + "directRecallAt10": 0.4, + "supportingRecallAt10": 0.111111 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 996, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.055556, + "recallAt10": 0.055556, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.095781, + "directRecallAt10": 0, + "supportingRecallAt10": 0.055556 + } + }, + "balancedV1": { + "deliveredTokens": 996, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.055556, + "recallAt10": 0.055556, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.095781, + "directRecallAt10": 0, + "supportingRecallAt10": 0.055556 + } + }, + "qarinahV2": { + "deliveredTokens": 996, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.055556, + "recallAt10": 0.055556, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.095781, + "directRecallAt10": 0, + "supportingRecallAt10": 0.055556 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 996, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.055556, + "recallAt10": 0.055556, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.095781, + "directRecallAt10": 0, + "supportingRecallAt10": 0.055556 + } + }, + "oracle": { + "deliveredTokens": 875, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.612308, + "directRecallAt10": 0.6, + "supportingRecallAt10": 0.166667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1940, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.156212, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + } + }, + "balancedV1": { + "deliveredTokens": 1858, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.143672, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + } + }, + "qarinahV2": { + "deliveredTokens": 1940, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.156212, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1940, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.156212, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + } + }, + "oracle": { + "deliveredTokens": 1931, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.277778, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.773113, + "directRecallAt10": 0.8, + "supportingRecallAt10": 0.277778 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 4000, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.277778, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.378188, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.277778 + } + }, + "balancedV1": { + "deliveredTokens": 3948, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.317756, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.222222 + } + }, + "qarinahV2": { + "deliveredTokens": 4000, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.277778, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.378188, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.277778 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 4000, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.277778, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.378188, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.277778 + } + }, + "oracle": { + "deliveredTokens": 3859, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.444444, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.94348, + "directRecallAt10": 1, + "supportingRecallAt10": 0.444444 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7977, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.222222, + "recallAt10": 0.5, + "precisionAt5": 0.8, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.554346, + "directRecallAt10": 0.4, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 7824, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.222222, + "recallAt10": 0.5, + "precisionAt5": 0.8, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.658001, + "directRecallAt10": 0.8, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 7977, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.222222, + "recallAt10": 0.5, + "precisionAt5": 0.8, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.554346, + "directRecallAt10": 0.4, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7977, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.222222, + "recallAt10": 0.5, + "precisionAt5": 0.8, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.554346, + "directRecallAt10": 0.4, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 7680, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.555556, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.555556 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.642222, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "matplotlib/matplotlib", + "instanceId": "matplotlib__matplotlib-26011", + "repositorySequence": 22, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 17, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.5, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.83276, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + }, + "balancedV1": { + "recallAt1": 0.055556, + "recallAt5": 0.222222, + "recallAt10": 0.444444, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.567004, + "directRecallAt10": 0, + "supportingRecallAt10": 0.444444 + }, + "qarinahV2": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.5, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.83276, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.5, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.83276, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + }, + "oracle": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.555556, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.555556 + } + }, + "volumes": { + "bm25Admitted": 6400, + "balancedV1": 6381, + "qarinahV2": 6400, + "qarinahV2NoGraph": 6400, + "oracle": 4698 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 453, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.442082, + "directRecallAt10": 1, + "supportingRecallAt10": 0.111111 + } + }, + "balancedV1": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.055556, + "recallAt10": 0.055556, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.055556 + } + }, + "qarinahV2": { + "deliveredTokens": 453, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.442082, + "directRecallAt10": 1, + "supportingRecallAt10": 0.111111 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 453, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.442082, + "directRecallAt10": 1, + "supportingRecallAt10": 0.111111 + } + }, + "oracle": { + "deliveredTokens": 453, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.111111 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.518493, + "directRecallAt10": 1, + "supportingRecallAt10": 0.166667 + } + }, + "balancedV1": { + "deliveredTokens": 868, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.111111 + } + }, + "qarinahV2": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.518493, + "directRecallAt10": 1, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 944, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.518493, + "directRecallAt10": 1, + "supportingRecallAt10": 0.166667 + } + }, + "oracle": { + "deliveredTokens": 964, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.631297, + "directRecallAt10": 1, + "supportingRecallAt10": 0.166667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.277778, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.643429, + "directRecallAt10": 1, + "supportingRecallAt10": 0.277778 + } + }, + "balancedV1": { + "deliveredTokens": 1924, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.39147, + "directRecallAt10": 0, + "supportingRecallAt10": 0.222222 + } + }, + "qarinahV2": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.277778, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.643429, + "directRecallAt10": 1, + "supportingRecallAt10": 0.277778 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.277778, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.643429, + "directRecallAt10": 1, + "supportingRecallAt10": 0.277778 + } + }, + "oracle": { + "deliveredTokens": 1956, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.333333, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.81067, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3915, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.333333, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.69437, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 3937, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.222222, + "recallAt10": 0.333333, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.479555, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 3915, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.333333, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.69437, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3915, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.333333, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.69437, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 3896, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.444444, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.909821, + "directRecallAt10": 1, + "supportingRecallAt10": 0.444444 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7968, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.5, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.83276, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 7981, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.222222, + "recallAt10": 0.444444, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.567004, + "directRecallAt10": 0, + "supportingRecallAt10": 0.444444 + } + }, + "qarinahV2": { + "deliveredTokens": 7968, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.5, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.83276, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7968, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.5, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.83276, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 7748, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.555556, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.555556 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.444549, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "matplotlib/matplotlib", + "instanceId": "matplotlib__matplotlib-26020", + "repositorySequence": 23, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.375906, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.463242, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.375906, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.375906, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 7181, + "balancedV1": 5853, + "qarinahV2": 7181, + "qarinahV2NoGraph": 7181, + "oracle": 1885 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 459, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 293, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 912, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 950, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 912, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 912, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 665, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1915, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.202107, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 1918, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 1915, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.202107, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1915, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.202107, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 1885, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3998, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.4018, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 3957, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.463242, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3998, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.4018, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3998, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.4018, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 1885, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.375906, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 7870, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.463242, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 7997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.375906, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.375906, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 1885, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.431812, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "mwaskom/seaborn", + "instanceId": "mwaskom__seaborn-3010", + "repositorySequence": 2, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 302, + "balancedV1": 302, + "qarinahV2": 302, + "qarinahV2NoGraph": 302, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 302, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 302, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 302, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 302, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 302, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 302, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 302, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 302, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 302, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 302, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 302, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 302, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 302, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 302, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 302, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 302, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 302, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 302, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 302, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 302, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.234877, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 4 + }, + { + "repository": "mwaskom/seaborn", + "instanceId": "mwaskom__seaborn-3190", + "repositorySequence": 3, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 1342, + "balancedV1": 1342, + "qarinahV2": 1342, + "qarinahV2NoGraph": 1342, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 302, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 302, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 302, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 302, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 302, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 302, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 302, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 302, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1342, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1342, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1342, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1342, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1342, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1342, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1342, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1342, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1342, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1342, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1342, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1342, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.552419, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 4 + }, + { + "repository": "mwaskom/seaborn", + "instanceId": "mwaskom__seaborn-3407", + "repositorySequence": 4, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1848, + "balancedV1": 1848, + "qarinahV2": 1848, + "qarinahV2NoGraph": 1848, + "oracle": 302 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 302, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 302, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 302, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 302, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 302, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 808, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 808, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 808, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 808, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 302, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1848, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1848, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1848, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1848, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 302, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1848, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1848, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1848, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1848, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 302, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1848, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1848, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1848, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1848, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 302, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.384731, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 4 + }, + { + "repository": "pallets/flask", + "instanceId": "pallets__flask-4992", + "repositorySequence": 2, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 113, + "balancedV1": 113, + "qarinahV2": 113, + "qarinahV2NoGraph": 113, + "oracle": 113 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 113, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.260396, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 3 + }, + { + "repository": "pallets/flask", + "instanceId": "pallets__flask-5063", + "repositorySequence": 3, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 449, + "balancedV1": 449, + "qarinahV2": 449, + "qarinahV2NoGraph": 449, + "oracle": 449 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 449, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 449, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 449, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 449, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 449, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 449, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 449, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 449, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 449, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 449, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 449, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 449, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 449, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 449, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 449, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 449, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 449, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 449, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 449, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 449, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 449, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 449, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 449, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 449, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 449, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.30875, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 3 + }, + { + "repository": "psf/requests", + "instanceId": "psf__requests-1963", + "repositorySequence": 2, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 253, + "balancedV1": 253, + "qarinahV2": 253, + "qarinahV2NoGraph": 253, + "oracle": 253 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 253, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.306579, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 6 + }, + { + "repository": "psf/requests", + "instanceId": "psf__requests-2148", + "repositorySequence": 3, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 493, + "balancedV1": 493, + "qarinahV2": 493, + "qarinahV2NoGraph": 493, + "oracle": 493 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.296145, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 6 + }, + { + "repository": "psf/requests", + "instanceId": "psf__requests-2317", + "repositorySequence": 4, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.757924, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.757924, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.757924, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.757924, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1133, + "balancedV1": 1133, + "qarinahV2": 1133, + "qarinahV2NoGraph": 1133, + "oracle": 1133 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.700276, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.700276, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.700276, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.700276, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 893, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.394809, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 893, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.394809, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 893, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.394809, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 893, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.394809, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1133, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.757924, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1133, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.757924, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1133, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.757924, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1133, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.757924, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1133, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1133, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.757924, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1133, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.757924, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1133, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.757924, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1133, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.757924, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1133, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1133, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.757924, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1133, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.757924, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1133, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.757924, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1133, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.757924, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1133, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.455714, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 6 + }, + { + "repository": "psf/requests", + "instanceId": "psf__requests-2674", + "repositorySequence": 5, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 4, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1392, + "balancedV1": 1392, + "qarinahV2": 1392, + "qarinahV2NoGraph": 1392, + "oracle": 1392 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 512, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 512, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 512, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 512, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 893, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 893, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 893, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 893, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 752, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1392, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1392, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1392, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1392, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1392, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1392, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1392, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1392, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1392, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1392, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1392, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1392, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1392, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1392, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1392, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.559459, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 6 + }, + { + "repository": "psf/requests", + "instanceId": "psf__requests-3362", + "repositorySequence": 6, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 5, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1570, + "balancedV1": 1570, + "qarinahV2": 1570, + "qarinahV2NoGraph": 1570, + "oracle": 1570 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.553146, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + }, + "balancedV1": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.553146, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.553146, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.553146, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + }, + "oracle": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.553146, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 880, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.553146, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + }, + "balancedV1": { + "deliveredTokens": 899, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.553146, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2": { + "deliveredTokens": 880, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.553146, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 880, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.553146, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + }, + "oracle": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.8, + "recallAt10": 0.8, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.868795, + "directRecallAt10": null, + "supportingRecallAt10": 0.8 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1570, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1570, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1570, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1570, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1570, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1570, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1570, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1570, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1570, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1570, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1570, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1570, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1570, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1570, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1570, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.423985, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "SINGLE_OR_NO_SUPPORT", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 6 + }, + { + "repository": "pydata/xarray", + "instanceId": "pydata__xarray-4094", + "repositorySequence": 2, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 215, + "balancedV1": 215, + "qarinahV2": 215, + "qarinahV2NoGraph": 215, + "oracle": 215 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.175403, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "NO_CODE_ENTITY_MATCH", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 5 + }, + { + "repository": "pydata/xarray", + "instanceId": "pydata__xarray-4248", + "repositorySequence": 3, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 672, + "balancedV1": 672, + "qarinahV2": 672, + "qarinahV2NoGraph": 672, + "oracle": 672 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 457, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 457, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 457, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 457, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 215, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 672, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 672, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 672, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 672, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 672, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 672, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 672, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 672, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 672, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 672, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 672, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 672, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 672, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 672, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 672, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 672, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 672, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 672, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 672, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 672, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.620313, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 5 + }, + { + "repository": "pydata/xarray", + "instanceId": "pydata__xarray-4493", + "repositorySequence": 4, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 947, + "balancedV1": 947, + "qarinahV2": 947, + "qarinahV2NoGraph": 947, + "oracle": 947 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 457, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 457, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 457, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 457, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 490, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 947, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 947, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 947, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 947, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 947, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 947, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 947, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 947, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 947, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 947, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 947, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 947, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 947, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 947, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 947, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 947, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 947, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 947, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 947, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 947, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.573958, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 5 + }, + { + "repository": "pydata/xarray", + "instanceId": "pydata__xarray-5131", + "repositorySequence": 5, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 4, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1600, + "balancedV1": 1600, + "qarinahV2": 1600, + "qarinahV2NoGraph": 1600, + "oracle": 1600 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 457, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 490, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 457, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 457, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39038, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 490, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 868, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 868, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 868, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 868, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 868, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1600, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1600, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1600, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1600, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1600, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1600, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1600, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1600, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1600, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1600, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1600, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1600, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1600, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1600, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1600, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.408365, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 5 + }, + { + "repository": "pylint-dev/pylint", + "instanceId": "pylint-dev__pylint-6506", + "repositorySequence": 2, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 287, + "balancedV1": 287, + "qarinahV2": 287, + "qarinahV2NoGraph": 287, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 287, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 287, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 287, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 287, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 287, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 287, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 287, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 287, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 287, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 287, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 287, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.4218, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "SINGLE_OR_NO_SUPPORT", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 6 + }, + { + "repository": "pylint-dev/pylint", + "instanceId": "pylint-dev__pylint-7080", + "repositorySequence": 3, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 770, + "balancedV1": 770, + "qarinahV2": 770, + "qarinahV2NoGraph": 770, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 483, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 483, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 483, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 483, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 770, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 770, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 770, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 770, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 770, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 770, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 770, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 770, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 770, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 770, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 770, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 770, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 770, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 770, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 770, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 770, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.320489, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 6 + }, + { + "repository": "pylint-dev/pylint", + "instanceId": "pylint-dev__pylint-7114", + "repositorySequence": 4, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 7016, + "balancedV1": 7016, + "qarinahV2": 7016, + "qarinahV2NoGraph": 7016, + "oracle": 6246 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 483, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 483, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 483, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 483, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 0, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 770, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 770, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 770, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 770, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 0, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 770, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 770, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 770, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 770, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 0, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 770, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 770, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 770, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 770, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 0, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7016, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 7016, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 7016, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7016, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 6246, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.508159, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 6 + }, + { + "repository": "pylint-dev/pylint", + "instanceId": "pylint-dev__pylint-7228", + "repositorySequence": 5, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 7497, + "balancedV1": 7497, + "qarinahV2": 7497, + "qarinahV2NoGraph": 7497, + "oracle": 483 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 483, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 481, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 483, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 483, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 483, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 770, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 964, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 770, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 770, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 483, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1251, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1251, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1251, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1251, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 483, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1251, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1251, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1251, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1251, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 483, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7497, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 7497, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 7497, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7497, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 483, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.485454, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 6 + }, + { + "repository": "pylint-dev/pylint", + "instanceId": "pylint-dev__pylint-7993", + "repositorySequence": 6, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 8596, + "balancedV1": 8596, + "qarinahV2": 8596, + "qarinahV2NoGraph": 8596, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 483, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 481, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 483, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 483, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 964, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 964, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 964, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 964, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1869, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1251, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1869, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1869, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2350, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 2350, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 2350, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2350, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2350, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 7497, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 2350, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2350, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.499677, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 6 + }, + { + "repository": "pytest-dev/pytest", + "instanceId": "pytest-dev__pytest-5413", + "repositorySequence": 4, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1044, + "balancedV1": 1044, + "qarinahV2": 1044, + "qarinahV2NoGraph": 1044, + "oracle": 1044 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1044, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.37875, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 9, + "noTemporalReturnedItems": 10 + }, + { + "repository": "pytest-dev/pytest", + "instanceId": "pytest-dev__pytest-5495", + "repositorySequence": 5, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 4, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1840, + "balancedV1": 1840, + "qarinahV2": 1840, + "qarinahV2NoGraph": 1840, + "oracle": 1840 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1840, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1840, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1840, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1840, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1840, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1840, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1840, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1840, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1840, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1840, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1840, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1840, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1840, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1840, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1840, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.397436, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 9, + "noTemporalReturnedItems": 10 + }, + { + "repository": "pytest-dev/pytest", + "instanceId": "pytest-dev__pytest-5692", + "repositorySequence": 6, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 5, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2084, + "balancedV1": 2084, + "qarinahV2": 2084, + "qarinahV2NoGraph": 2084, + "oracle": 2084 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.553146, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + }, + "balancedV1": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.553146, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.553146, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.553146, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + }, + "oracle": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.553146, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.553146, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + }, + "balancedV1": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.553146, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.553146, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.553146, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + }, + "oracle": { + "deliveredTokens": 664, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.6, + "recallAt10": 0.6, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.722727, + "directRecallAt10": null, + "supportingRecallAt10": 0.6 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.8, + "recallAt10": 0.8, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.868795, + "directRecallAt10": null, + "supportingRecallAt10": 0.8 + } + }, + "balancedV1": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.8, + "recallAt10": 0.8, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.868795, + "directRecallAt10": null, + "supportingRecallAt10": 0.8 + } + }, + "qarinahV2": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.8, + "recallAt10": 0.8, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.868795, + "directRecallAt10": null, + "supportingRecallAt10": 0.8 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.8, + "recallAt10": 0.8, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.868795, + "directRecallAt10": null, + "supportingRecallAt10": 0.8 + } + }, + "oracle": { + "deliveredTokens": 1840, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.8, + "recallAt10": 0.8, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.868795, + "directRecallAt10": null, + "supportingRecallAt10": 0.8 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2084, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2084, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2084, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2084, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2084, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2084, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2084, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2084, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2084, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2084, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.349569, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 8, + "noTemporalReturnedItems": 10 + }, + { + "repository": "pytest-dev/pytest", + "instanceId": "pytest-dev__pytest-6116", + "repositorySequence": 7, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 6, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2464, + "balancedV1": 2464, + "qarinahV2": 2464, + "qarinahV2NoGraph": 2464, + "oracle": 2464 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.493523, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.493523, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.493523, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 420, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.493523, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 494, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.493523, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 664, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.644824, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.493523, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 664, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.644824, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 664, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.644824, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 800, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.644824, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.775148, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 1668, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 0.833333, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.892211, + "directRecallAt10": null, + "supportingRecallAt10": 0.833333 + } + }, + "qarinahV2": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.775148, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.775148, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 1668, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 0.833333, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.892211, + "directRecallAt10": null, + "supportingRecallAt10": 0.833333 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2464, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2464, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2464, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2464, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2464, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2464, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2464, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2464, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2464, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2464, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.214286, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "NO_CODE_ENTITY_MATCH", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "pytest-dev/pytest", + "instanceId": "pytest-dev__pytest-7168", + "repositorySequence": 8, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 7, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2766, + "balancedV1": 2766, + "qarinahV2": 2766, + "qarinahV2NoGraph": 2766, + "oracle": 2766 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 494, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.448304, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + }, + "balancedV1": { + "deliveredTokens": 494, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.448304, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + }, + "qarinahV2": { + "deliveredTokens": 494, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.448304, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 494, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.448304, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + }, + "oracle": { + "deliveredTokens": 494, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.448304, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 982, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.428571, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.585742, + "directRecallAt10": null, + "supportingRecallAt10": 0.428571 + } + }, + "balancedV1": { + "deliveredTokens": 930, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.448304, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + }, + "qarinahV2": { + "deliveredTokens": 982, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.428571, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.585742, + "directRecallAt10": null, + "supportingRecallAt10": 0.428571 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 982, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.428571, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.585742, + "directRecallAt10": null, + "supportingRecallAt10": 0.428571 + } + }, + "oracle": { + "deliveredTokens": 800, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.428571, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.585742, + "directRecallAt10": null, + "supportingRecallAt10": 0.428571 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1914, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.571429, + "recallAt10": 0.571429, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.704125, + "directRecallAt10": null, + "supportingRecallAt10": 0.571429 + } + }, + "balancedV1": { + "deliveredTokens": 1914, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.571429, + "recallAt10": 0.571429, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.704125, + "directRecallAt10": null, + "supportingRecallAt10": 0.571429 + } + }, + "qarinahV2": { + "deliveredTokens": 1914, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.571429, + "recallAt10": 0.571429, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.704125, + "directRecallAt10": null, + "supportingRecallAt10": 0.571429 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1914, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.571429, + "recallAt10": 0.571429, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.704125, + "directRecallAt10": null, + "supportingRecallAt10": 0.571429 + } + }, + "oracle": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 0.857143, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.908375, + "directRecallAt10": null, + "supportingRecallAt10": 0.857143 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2766, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2766, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2766, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2766, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2766, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2766, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2766, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2766, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2766, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2766, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.327971, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 8, + "noTemporalReturnedItems": 10 + }, + { + "repository": "pytest-dev/pytest", + "instanceId": "pytest-dev__pytest-7220", + "repositorySequence": 9, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 8, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4932, + "balancedV1": 4932, + "qarinahV2": 4932, + "qarinahV2NoGraph": 4932, + "oracle": 4932 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 358, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.412532, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 358, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.412532, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2": { + "deliveredTokens": 358, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.412532, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 358, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.412532, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 494, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.412532, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 910, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.412532, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 910, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.412532, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2": { + "deliveredTokens": 910, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.412532, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 910, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.412532, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 800, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.375, + "recallAt10": 0.375, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.539003, + "directRecallAt10": null, + "supportingRecallAt10": 0.375 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1966, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.64794, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 1966, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.64794, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 1966, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.64794, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1966, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.64794, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 0.75, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.835891, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3944, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 0.625, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.745791, + "directRecallAt10": null, + "supportingRecallAt10": 0.625 + } + }, + "balancedV1": { + "deliveredTokens": 3944, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 0.625, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.745791, + "directRecallAt10": null, + "supportingRecallAt10": 0.625 + } + }, + "qarinahV2": { + "deliveredTokens": 3944, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 0.625, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.745791, + "directRecallAt10": null, + "supportingRecallAt10": 0.625 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3944, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 0.625, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.745791, + "directRecallAt10": null, + "supportingRecallAt10": 0.625 + } + }, + "oracle": { + "deliveredTokens": 3892, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 0.75, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.835891, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 4932, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 4932, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 4932, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 4932, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 4932, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.43157, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "pytest-dev/pytest", + "instanceId": "pytest-dev__pytest-7373", + "repositorySequence": 10, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 9, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 5648, + "balancedV1": 5648, + "qarinahV2": 5648, + "qarinahV2NoGraph": 5648, + "oracle": 5648 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 416, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.383343, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + }, + "balancedV1": { + "deliveredTokens": 416, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.383343, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + }, + "qarinahV2": { + "deliveredTokens": 416, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.383343, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 416, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.383343, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + }, + "oracle": { + "deliveredTokens": 494, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.383343, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 960, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.383343, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + }, + "balancedV1": { + "deliveredTokens": 926, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.383343, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + }, + "qarinahV2": { + "deliveredTokens": 960, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.383343, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 960, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.383343, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + }, + "oracle": { + "deliveredTokens": 800, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.500866, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 0.555556, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.693022, + "directRecallAt10": null, + "supportingRecallAt10": 0.555556 + } + }, + "balancedV1": { + "deliveredTokens": 1948, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.444444, + "recallAt10": 0.444444, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.602094, + "directRecallAt10": null, + "supportingRecallAt10": 0.444444 + } + }, + "qarinahV2": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 0.555556, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.693022, + "directRecallAt10": null, + "supportingRecallAt10": 0.555556 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 0.555556, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.693022, + "directRecallAt10": null, + "supportingRecallAt10": 0.555556 + } + }, + "oracle": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 0.666667, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.776747, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3482, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 0.888889, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.929244, + "directRecallAt10": null, + "supportingRecallAt10": 0.888889 + } + }, + "balancedV1": { + "deliveredTokens": 3482, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 0.888889, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.929244, + "directRecallAt10": null, + "supportingRecallAt10": 0.888889 + } + }, + "qarinahV2": { + "deliveredTokens": 3482, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 0.888889, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.929244, + "directRecallAt10": null, + "supportingRecallAt10": 0.888889 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3482, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 0.888889, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.929244, + "directRecallAt10": null, + "supportingRecallAt10": 0.888889 + } + }, + "oracle": { + "deliveredTokens": 3892, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 0.666667, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.776747, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 5648, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 5648, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 5648, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 5648, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 5648, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.416883, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "pytest-dev/pytest", + "instanceId": "pytest-dev__pytest-7432", + "repositorySequence": 11, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 10, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 5969, + "balancedV1": 5969, + "qarinahV2": 5969, + "qarinahV2NoGraph": 5969, + "oracle": 5969 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 435, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "balancedV1": { + "deliveredTokens": 435, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2": { + "deliveredTokens": 435, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 435, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "oracle": { + "deliveredTokens": 435, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 945, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "balancedV1": { + "deliveredTokens": 945, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2": { + "deliveredTokens": 945, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 945, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.2 + } + }, + "oracle": { + "deliveredTokens": 815, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.3, + "recallAt10": 0.3, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.3 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1905, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + }, + "balancedV1": { + "deliveredTokens": 1905, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2": { + "deliveredTokens": 1905, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1905, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.4 + } + }, + "oracle": { + "deliveredTokens": 1989, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 0.6, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.6 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3941, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 3803, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 0.9, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.936379, + "directRecallAt10": null, + "supportingRecallAt10": 0.9 + } + }, + "qarinahV2": { + "deliveredTokens": 3941, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3941, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 3911, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 0.6, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.6 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 5969, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 5969, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 5969, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 5969, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 5969, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.593189, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "pytest-dev/pytest", + "instanceId": "pytest-dev__pytest-7490", + "repositorySequence": 12, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 10, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.812595, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + }, + "balancedV1": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.812595, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.812595, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.812595, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 6109, + "balancedV1": 6109, + "qarinahV2": 6109, + "qarinahV2NoGraph": 6109, + "oracle": 5979 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 498, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 498, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 498, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 498, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 368, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 910, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 910, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 910, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 910, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 955, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.631297, + "directRecallAt10": 1, + "supportingRecallAt10": 0.272727 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1880, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.544292, + "directRecallAt10": 1, + "supportingRecallAt10": 0.363636 + } + }, + "balancedV1": { + "deliveredTokens": 1880, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.544292, + "directRecallAt10": 1, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2": { + "deliveredTokens": 1880, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.544292, + "directRecallAt10": 1, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1880, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.544292, + "directRecallAt10": 1, + "supportingRecallAt10": 0.363636 + } + }, + "oracle": { + "deliveredTokens": 1999, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.545455, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.81067, + "directRecallAt10": 1, + "supportingRecallAt10": 0.545455 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3943, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.818182, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.781814, + "directRecallAt10": 1, + "supportingRecallAt10": 0.818182 + } + }, + "balancedV1": { + "deliveredTokens": 3943, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.818182, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.781814, + "directRecallAt10": 1, + "supportingRecallAt10": 0.818182 + } + }, + "qarinahV2": { + "deliveredTokens": 3943, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.818182, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.781814, + "directRecallAt10": 1, + "supportingRecallAt10": 0.818182 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3943, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.818182, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.781814, + "directRecallAt10": 1, + "supportingRecallAt10": 0.818182 + } + }, + "oracle": { + "deliveredTokens": 3843, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.636364, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.861611, + "directRecallAt10": 1, + "supportingRecallAt10": 0.636364 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 6223, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.812595, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + }, + "balancedV1": { + "deliveredTokens": 6223, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.812595, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + }, + "qarinahV2": { + "deliveredTokens": 6223, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.812595, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 6223, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.812595, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + }, + "oracle": { + "deliveredTokens": 6223, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.340618, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "NO_CODE_ENTITY_MATCH", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "pytest-dev/pytest", + "instanceId": "pytest-dev__pytest-8365", + "repositorySequence": 13, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 12, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.833333, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.833333 + }, + "balancedV1": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.833333, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.833333 + }, + "qarinahV2": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.833333, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.833333 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.833333, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.833333 + }, + "oracle": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.833333, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.833333 + } + }, + "volumes": { + "bm25Admitted": 11616, + "balancedV1": 11490, + "qarinahV2": 11616, + "qarinahV2NoGraph": 11616, + "oracle": 5979 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 435, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "balancedV1": { + "deliveredTokens": 498, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2": { + "deliveredTokens": 435, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 435, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "oracle": { + "deliveredTokens": 435, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 960, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "balancedV1": { + "deliveredTokens": 970, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2": { + "deliveredTokens": 960, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 960, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "oracle": { + "deliveredTokens": 815, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1889, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.416667, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.416667 + } + }, + "balancedV1": { + "deliveredTokens": 1899, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.416667, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.416667 + } + }, + "qarinahV2": { + "deliveredTokens": 1889, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.416667, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.416667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1889, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.416667, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.416667 + } + }, + "oracle": { + "deliveredTokens": 1999, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.5, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3925, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.5, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 3877, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.5, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 3925, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.5, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3925, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.5, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 3911, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.5, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 6223, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.833333, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.833333 + } + }, + "balancedV1": { + "deliveredTokens": 7982, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.5, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 6223, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.833333, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.833333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 6223, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.833333, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.833333 + } + }, + "oracle": { + "deliveredTokens": 6223, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.833333, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.833333 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.38545, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "pytest-dev/pytest", + "instanceId": "pytest-dev__pytest-8906", + "repositorySequence": 14, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 12, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.769231, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.694356, + "directRecallAt10": 0, + "supportingRecallAt10": 0.769231 + }, + "balancedV1": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.769231, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.694356, + "directRecallAt10": 0, + "supportingRecallAt10": 0.769231 + }, + "qarinahV2": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.769231, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.694356, + "directRecallAt10": 0, + "supportingRecallAt10": 0.769231 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.769231, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.694356, + "directRecallAt10": 0, + "supportingRecallAt10": 0.769231 + }, + "oracle": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.769231, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.769231 + } + }, + "volumes": { + "bm25Admitted": 11777, + "balancedV1": 11777, + "qarinahV2": 11777, + "qarinahV2NoGraph": 11777, + "oracle": 5979 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 498, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.153846, + "recallAt10": 0.153846, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.153846 + } + }, + "balancedV1": { + "deliveredTokens": 498, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.153846, + "recallAt10": 0.153846, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.153846 + } + }, + "qarinahV2": { + "deliveredTokens": 498, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.153846, + "recallAt10": 0.153846, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.153846 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 498, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.153846, + "recallAt10": 0.153846, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.153846 + } + }, + "oracle": { + "deliveredTokens": 435, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.153846, + "recallAt10": 0.153846, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.153846 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 899, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.230769, + "recallAt10": 0.230769, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.478475, + "directRecallAt10": 1, + "supportingRecallAt10": 0.230769 + } + }, + "balancedV1": { + "deliveredTokens": 899, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.230769, + "recallAt10": 0.230769, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.478475, + "directRecallAt10": 1, + "supportingRecallAt10": 0.230769 + } + }, + "qarinahV2": { + "deliveredTokens": 899, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.230769, + "recallAt10": 0.230769, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.478475, + "directRecallAt10": 1, + "supportingRecallAt10": 0.230769 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 899, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.230769, + "recallAt10": 0.230769, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.478475, + "directRecallAt10": 1, + "supportingRecallAt10": 0.230769 + } + }, + "oracle": { + "deliveredTokens": 815, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.230769, + "recallAt10": 0.230769, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.631297, + "directRecallAt10": 1, + "supportingRecallAt10": 0.230769 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1936, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.384615, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.568829, + "directRecallAt10": 1, + "supportingRecallAt10": 0.384615 + } + }, + "balancedV1": { + "deliveredTokens": 1936, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.384615, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.568829, + "directRecallAt10": 1, + "supportingRecallAt10": 0.384615 + } + }, + "qarinahV2": { + "deliveredTokens": 1936, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.384615, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.568829, + "directRecallAt10": 1, + "supportingRecallAt10": 0.384615 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1936, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.384615, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.568829, + "directRecallAt10": 1, + "supportingRecallAt10": 0.384615 + } + }, + "oracle": { + "deliveredTokens": 1999, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.461538, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.81067, + "directRecallAt10": 1, + "supportingRecallAt10": 0.461538 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3988, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.384615, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.450589, + "directRecallAt10": 0, + "supportingRecallAt10": 0.384615 + } + }, + "balancedV1": { + "deliveredTokens": 3964, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.692308, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.742189, + "directRecallAt10": 1, + "supportingRecallAt10": 0.692308 + } + }, + "qarinahV2": { + "deliveredTokens": 3988, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.384615, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.450589, + "directRecallAt10": 0, + "supportingRecallAt10": 0.384615 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3988, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.384615, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.450589, + "directRecallAt10": 0, + "supportingRecallAt10": 0.384615 + } + }, + "oracle": { + "deliveredTokens": 3911, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.461538, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.81067, + "directRecallAt10": 1, + "supportingRecallAt10": 0.461538 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7889, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.461538, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.505026, + "directRecallAt10": 0, + "supportingRecallAt10": 0.461538 + } + }, + "balancedV1": { + "deliveredTokens": 7889, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.461538, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.505026, + "directRecallAt10": 0, + "supportingRecallAt10": 0.461538 + } + }, + "qarinahV2": { + "deliveredTokens": 7889, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.461538, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.505026, + "directRecallAt10": 0, + "supportingRecallAt10": 0.461538 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7889, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.461538, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.505026, + "directRecallAt10": 0, + "supportingRecallAt10": 0.461538 + } + }, + "oracle": { + "deliveredTokens": 6754, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.769231, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.769231 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.436248, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "pytest-dev/pytest", + "instanceId": "pytest-dev__pytest-9359", + "repositorySequence": 15, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 14, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.714286, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.714286 + }, + "balancedV1": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.714286, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.714286 + }, + "qarinahV2": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.714286, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.714286 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.714286, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.714286 + }, + "oracle": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.714286, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.714286 + } + }, + "volumes": { + "bm25Admitted": 11942, + "balancedV1": 11750, + "qarinahV2": 11942, + "qarinahV2NoGraph": 11942, + "oracle": 5979 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 498, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "balancedV1": { + "deliveredTokens": 368, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2": { + "deliveredTokens": 498, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 498, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "oracle": { + "deliveredTokens": 435, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 960, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "balancedV1": { + "deliveredTokens": 912, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.214286, + "recallAt10": 0.214286, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.214286 + } + }, + "qarinahV2": { + "deliveredTokens": 960, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 960, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "oracle": { + "deliveredTokens": 815, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.214286, + "recallAt10": 0.214286, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.214286 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1985, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + }, + "balancedV1": { + "deliveredTokens": 1872, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.357143, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.357143 + } + }, + "qarinahV2": { + "deliveredTokens": 1985, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1985, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + }, + "oracle": { + "deliveredTokens": 1999, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.428571, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.428571 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3823, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.571429, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.870125, + "directRecallAt10": null, + "supportingRecallAt10": 0.571429 + } + }, + "balancedV1": { + "deliveredTokens": 3823, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.571429, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.870125, + "directRecallAt10": null, + "supportingRecallAt10": 0.571429 + } + }, + "qarinahV2": { + "deliveredTokens": 3823, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.571429, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.870125, + "directRecallAt10": null, + "supportingRecallAt10": 0.571429 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3823, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.571429, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.870125, + "directRecallAt10": null, + "supportingRecallAt10": 0.571429 + } + }, + "oracle": { + "deliveredTokens": 3911, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.428571, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.428571 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7897, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + }, + "balancedV1": { + "deliveredTokens": 7920, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.428571, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.428571 + } + }, + "qarinahV2": { + "deliveredTokens": 7897, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7897, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + }, + "oracle": { + "deliveredTokens": 7298, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.714286, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.714286 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "DIRECTLY_SUPPORTED", + "decision": "ACCEPT_DIRECT", + "score": 0.692373, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "CONSERVATIVE_DIRECT_THRESHOLD_MET" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "pytest-dev/pytest", + "instanceId": "pytest-dev__pytest-11143", + "repositorySequence": 16, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 14, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.82599, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + }, + "balancedV1": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.790776, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.82599, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.82599, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + }, + "oracle": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "volumes": { + "bm25Admitted": 12732, + "balancedV1": 12418, + "qarinahV2": 12732, + "qarinahV2NoGraph": 12732, + "oracle": 6175 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 416, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.133333, + "recallAt10": 0.133333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.133333 + } + }, + "balancedV1": { + "deliveredTokens": 368, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.133333, + "recallAt10": 0.133333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.133333 + } + }, + "qarinahV2": { + "deliveredTokens": 416, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.133333, + "recallAt10": 0.133333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.133333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 416, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.133333, + "recallAt10": 0.133333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.133333 + } + }, + "oracle": { + "deliveredTokens": 435, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.133333, + "recallAt10": 0.133333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.249242, + "directRecallAt10": 0, + "supportingRecallAt10": 0.133333 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 912, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.066667, + "recallAt10": 0.066667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.066667 + } + }, + "balancedV1": { + "deliveredTokens": 912, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.325653, + "directRecallAt10": 0, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2": { + "deliveredTokens": 912, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.066667, + "recallAt10": 0.066667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.066667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 912, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.066667, + "recallAt10": 0.066667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.152822, + "directRecallAt10": 0, + "supportingRecallAt10": 0.066667 + } + }, + "oracle": { + "deliveredTokens": 945, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.133333, + "recallAt10": 0.133333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.554886, + "directRecallAt10": 1, + "supportingRecallAt10": 0.133333 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1962, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.325653, + "directRecallAt10": 0, + "supportingRecallAt10": 0.2 + } + }, + "balancedV1": { + "deliveredTokens": 1824, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.266667, + "recallAt10": 0.266667, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.39147, + "directRecallAt10": 0, + "supportingRecallAt10": 0.266667 + } + }, + "qarinahV2": { + "deliveredTokens": 1962, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.325653, + "directRecallAt10": 0, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1962, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.325653, + "directRecallAt10": 0, + "supportingRecallAt10": 0.2 + } + }, + "oracle": { + "deliveredTokens": 1999, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.4, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.81067, + "directRecallAt10": 1, + "supportingRecallAt10": 0.4 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3995, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.466667, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.708789, + "directRecallAt10": 1, + "supportingRecallAt10": 0.466667 + } + }, + "balancedV1": { + "deliveredTokens": 3775, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.466667, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.664839, + "directRecallAt10": 1, + "supportingRecallAt10": 0.466667 + } + }, + "qarinahV2": { + "deliveredTokens": 3995, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.466667, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.708789, + "directRecallAt10": 1, + "supportingRecallAt10": 0.466667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3995, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.466667, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.708789, + "directRecallAt10": 1, + "supportingRecallAt10": 0.466667 + } + }, + "oracle": { + "deliveredTokens": 3911, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.4, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.81067, + "directRecallAt10": 1, + "supportingRecallAt10": 0.4 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 8000, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.266667, + "recallAt10": 0.266667, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.39147, + "directRecallAt10": 0, + "supportingRecallAt10": 0.266667 + } + }, + "balancedV1": { + "deliveredTokens": 7965, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.4, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.505026, + "directRecallAt10": 0, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2": { + "deliveredTokens": 8000, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.266667, + "recallAt10": 0.266667, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.39147, + "directRecallAt10": 0, + "supportingRecallAt10": 0.266667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 8000, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.266667, + "recallAt10": 0.266667, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.39147, + "directRecallAt10": 0, + "supportingRecallAt10": 0.266667 + } + }, + "oracle": { + "deliveredTokens": 7679, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.37197, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "pytest-dev/pytest", + "instanceId": "pytest-dev__pytest-11148", + "repositorySequence": 17, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 16, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.0625, + "recallAt5": 0.3125, + "recallAt10": 0.625, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.625 + }, + "balancedV1": { + "recallAt1": 0.0625, + "recallAt5": 0.3125, + "recallAt10": 0.625, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.625 + }, + "qarinahV2": { + "recallAt1": 0.0625, + "recallAt5": 0.3125, + "recallAt10": 0.625, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.625 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.0625, + "recallAt5": 0.3125, + "recallAt10": 0.625, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.625 + }, + "oracle": { + "recallAt1": 0.0625, + "recallAt5": 0.3125, + "recallAt10": 0.625, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.625 + } + }, + "volumes": { + "bm25Admitted": 13521, + "balancedV1": 13262, + "qarinahV2": 13521, + "qarinahV2NoGraph": 13521, + "oracle": 6638 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 435, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + }, + "balancedV1": { + "deliveredTokens": 368, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + }, + "qarinahV2": { + "deliveredTokens": 435, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 435, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + }, + "oracle": { + "deliveredTokens": 435, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 912, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.0625, + "recallAt10": 0.0625, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.0625 + } + }, + "balancedV1": { + "deliveredTokens": 992, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.1875, + "recallAt10": 0.1875, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.1875 + } + }, + "qarinahV2": { + "deliveredTokens": 912, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.0625, + "recallAt10": 0.0625, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.0625 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 912, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.0625, + "recallAt10": 0.0625, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.0625 + } + }, + "oracle": { + "deliveredTokens": 815, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.1875, + "recallAt10": 0.1875, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.1875 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1952, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.1875, + "recallAt10": 0.1875, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.1875 + } + }, + "balancedV1": { + "deliveredTokens": 1999, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + }, + "qarinahV2": { + "deliveredTokens": 1952, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.1875, + "recallAt10": 0.1875, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.1875 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1952, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.1875, + "recallAt10": 0.1875, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.1875 + } + }, + "oracle": { + "deliveredTokens": 1810, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.1875, + "recallAt10": 0.1875, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.1875 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3913, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.3125, + "recallAt10": 0.375, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.375 + } + }, + "balancedV1": { + "deliveredTokens": 3823, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.3125, + "recallAt10": 0.375, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.375 + } + }, + "qarinahV2": { + "deliveredTokens": 3913, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.3125, + "recallAt10": 0.375, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.375 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3913, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.3125, + "recallAt10": 0.375, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.72733, + "directRecallAt10": null, + "supportingRecallAt10": 0.375 + } + }, + "oracle": { + "deliveredTokens": 3976, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.563788, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7904, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.3125, + "recallAt10": 0.3125, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.3125 + } + }, + "balancedV1": { + "deliveredTokens": 7822, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.3125, + "recallAt10": 0.625, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.625 + } + }, + "qarinahV2": { + "deliveredTokens": 7904, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.3125, + "recallAt10": 0.3125, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.3125 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7904, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.3125, + "recallAt10": 0.3125, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.3125 + } + }, + "oracle": { + "deliveredTokens": 7794, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.3125, + "recallAt10": 0.625, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.625 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.380516, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "scikit-learn/scikit-learn", + "instanceId": "scikit-learn__scikit-learn-12471", + "repositorySequence": 6, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1778, + "balancedV1": 1778, + "qarinahV2": 1778, + "qarinahV2NoGraph": 1778, + "oracle": 475 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 475, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 893, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 893, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 893, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 893, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 475, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 475, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 475, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1778, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 475, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.492532, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 10, + "noTemporalReturnedItems": 10 + }, + { + "repository": "scikit-learn/scikit-learn", + "instanceId": "scikit-learn__scikit-learn-13142", + "repositorySequence": 7, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.386853, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.386853, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.386853, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2374, + "balancedV1": 2374, + "qarinahV2": 2374, + "qarinahV2NoGraph": 2374, + "oracle": 198 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 198, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 794, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 794, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 794, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 794, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 198, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1931, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1932, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1931, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1931, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 198, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2374, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.386853, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2374, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2374, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.386853, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2374, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.386853, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 198, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2374, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.386853, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2374, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2374, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.386853, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2374, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.386853, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 198, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.529386, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 9, + "noTemporalReturnedItems": 10 + }, + { + "repository": "scikit-learn/scikit-learn", + "instanceId": "scikit-learn__scikit-learn-13241", + "repositorySequence": 8, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 3009, + "balancedV1": 3009, + "qarinahV2": 3009, + "qarinahV2NoGraph": 3009, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 418, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 443, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 418, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 418, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 794, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 833, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 794, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 794, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1847, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1847, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1847, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1847, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3009, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3009, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3009, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3009, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3009, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3009, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3009, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3009, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.522727, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "scikit-learn/scikit-learn", + "instanceId": "scikit-learn__scikit-learn-13439", + "repositorySequence": 9, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 3270, + "balancedV1": 3270, + "qarinahV2": 3270, + "qarinahV2NoGraph": 3270, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 418, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 443, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 418, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 418, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 857, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 833, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 857, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 857, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1910, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1871, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1910, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1910, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3270, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3270, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3270, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3270, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3270, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3270, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3270, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3270, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.491034, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 8, + "noTemporalReturnedItems": 10 + }, + { + "repository": "scikit-learn/scikit-learn", + "instanceId": "scikit-learn__scikit-learn-13496", + "repositorySequence": 10, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 3631, + "balancedV1": 3631, + "qarinahV2": 3631, + "qarinahV2NoGraph": 3631, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 442, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 443, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 442, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 442, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 885, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 833, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 885, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 885, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1742, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1871, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1742, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1742, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3631, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3631, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3631, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3631, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3631, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3631, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3631, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3631, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.425133, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "scikit-learn/scikit-learn", + "instanceId": "scikit-learn__scikit-learn-13497", + "repositorySequence": 11, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 4028, + "balancedV1": 4028, + "qarinahV2": 4028, + "qarinahV2NoGraph": 4028, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 397, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 443, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 397, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 397, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 993, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 833, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 993, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 993, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1941, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1917, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1941, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1941, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3610, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3610, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3610, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3610, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 4028, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 4028, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 4028, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 4028, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.584, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "scikit-learn/scikit-learn", + "instanceId": "scikit-learn__scikit-learn-13584", + "repositorySequence": 12, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.356207, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.315465, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.356207, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.356207, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3900, + "balancedV1": 3900, + "qarinahV2": 3900, + "qarinahV2NoGraph": 3900, + "oracle": 418 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 442, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 331, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 442, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 442, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 972, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 966, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 972, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 972, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1910, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1871, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1910, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1910, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3900, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.356207, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3900, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.315465, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3900, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.356207, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3900, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.356207, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 4161, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.356207, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 4161, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.315465, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 4161, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.356207, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 4161, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.356207, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.540645, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "scikit-learn/scikit-learn", + "instanceId": "scikit-learn__scikit-learn-13779", + "repositorySequence": 13, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.25, + "ndcgAt10": 0.357449, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.62784, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.25, + "ndcgAt10": 0.357449, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.25, + "ndcgAt10": 0.357449, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3918, + "balancedV1": 3918, + "qarinahV2": 3918, + "qarinahV2NoGraph": 3918, + "oracle": 595 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 442, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 443, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.826235, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 442, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 442, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 198, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.826235, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.521296, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 595, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1964, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.118613, + "directRecallAt10": 0, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 1917, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.62784, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1964, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.118613, + "directRecallAt10": 0, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1964, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.118613, + "directRecallAt10": 0, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 595, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3918, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.25, + "ndcgAt10": 0.357449, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3918, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.62784, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3918, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.25, + "ndcgAt10": 0.357449, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3918, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.25, + "ndcgAt10": 0.357449, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 595, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 4051, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.25, + "ndcgAt10": 0.357449, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 4312, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.62784, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 4312, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.25, + "ndcgAt10": 0.357449, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 4312, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.25, + "ndcgAt10": 0.357449, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 595, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.528571, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "scikit-learn/scikit-learn", + "instanceId": "scikit-learn__scikit-learn-14087", + "repositorySequence": 14, + "positiveUnderStructuralOracle": true, + "directRecords": 2, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.268209, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.666667 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.598309, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.268209, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.268209, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.666667 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3943, + "balancedV1": 3880, + "qarinahV2": 3943, + "qarinahV2NoGraph": 3943, + "oracle": 847 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 512, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 405, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.907284, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 512, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 512, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 405, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.907284, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 996, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.350985, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 996, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 996, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 847, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1989, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.092716, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 1994, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.646052, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1989, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.092716, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1989, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.092716, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 847, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3943, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.268209, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 3880, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.598309, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3943, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.268209, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3943, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.268209, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 847, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 4519, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.268209, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 4519, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.598309, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 4519, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.268209, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 4519, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.268209, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 847, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.534252, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "scikit-learn/scikit-learn", + "instanceId": "scikit-learn__scikit-learn-14092", + "repositorySequence": 15, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.333333, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4192, + "balancedV1": 4192, + "qarinahV2": 4192, + "qarinahV2NoGraph": 4192, + "oracle": 245 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 452, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 443, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 452, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 452, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 245, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 974, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 974, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 974, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 245, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1965, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1871, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1965, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1965, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 245, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3975, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3982, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.333333, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3975, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3975, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 245, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 5155, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 5155, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.333333, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 5155, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 5155, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 245, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.355208, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "scikit-learn/scikit-learn", + "instanceId": "scikit-learn__scikit-learn-14894", + "repositorySequence": 16, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 4861, + "balancedV1": 4798, + "qarinahV2": 4861, + "qarinahV2NoGraph": 4861, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 512, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 405, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 512, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 512, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 997, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 984, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 997, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 997, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 2000, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1944, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 2000, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2000, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3975, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3968, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3975, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3975, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 5985, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 5985, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 5985, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 5985, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.611692, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "scikit-learn/scikit-learn", + "instanceId": "scikit-learn__scikit-learn-14983", + "repositorySequence": 17, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 4816, + "balancedV1": 4863, + "qarinahV2": 4816, + "qarinahV2NoGraph": 4816, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 512, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 459, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 512, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 512, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 997, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 984, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 997, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 997, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1893, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1981, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1893, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1893, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3986, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3973, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3986, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3986, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 6471, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 6471, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 6471, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 6471, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.602317, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "scikit-learn/scikit-learn", + "instanceId": "scikit-learn__scikit-learn-15512", + "repositorySequence": 18, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 5230, + "balancedV1": 5032, + "qarinahV2": 5230, + "qarinahV2NoGraph": 5230, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 512, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 482, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 512, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 512, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 957, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 984, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 957, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 957, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1989, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1941, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1989, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1989, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3962, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3978, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3962, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3962, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 6867, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 6867, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 6867, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 6867, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.439732, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "scikit-learn/scikit-learn", + "instanceId": "scikit-learn__scikit-learn-15535", + "repositorySequence": 19, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 4783, + "balancedV1": 4448, + "qarinahV2": 4783, + "qarinahV2NoGraph": 4783, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 443, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 443, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 443, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 443, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 979, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 984, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 979, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 979, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1990, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1908, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1990, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1990, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3949, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3985, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3949, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3949, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7403, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 7403, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 7403, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7403, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.577632, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "scikit-learn/scikit-learn", + "instanceId": "scikit-learn__scikit-learn-25500", + "repositorySequence": 20, + "positiveUnderStructuralOracle": true, + "directRecords": 6, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.714286, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.688113, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.714286 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.428571, + "recallAt10": 0.714286, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.593986, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.714286 + }, + "qarinahV2": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.714286, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.688113, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.714286 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.714286, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.688113, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.714286 + }, + "oracle": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4981, + "balancedV1": 5138, + "qarinahV2": 4981, + "qarinahV2NoGraph": 4981, + "oracle": 3009 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 442, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.292759, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2": { + "deliveredTokens": 442, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 442, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 475, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.292759, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.142857 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 957, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.354329, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.285714 + } + }, + "balancedV1": { + "deliveredTokens": 984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.18471, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2": { + "deliveredTokens": 957, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.354329, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.285714 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 957, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.354329, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.285714 + } + }, + "oracle": { + "deliveredTokens": 880, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.428571, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.623849, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.428571 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1963, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.439139, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.285714 + } + }, + "balancedV1": { + "deliveredTokens": 1965, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.428571, + "recallAt10": 0.428571, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.457174, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.428571 + } + }, + "qarinahV2": { + "deliveredTokens": 1963, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.439139, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.285714 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1963, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.439139, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.285714 + } + }, + "oracle": { + "deliveredTokens": 1905, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.571429, + "recallAt10": 0.571429, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.749934, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.571429 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3888, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.571429, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.599984, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.571429 + } + }, + "balancedV1": { + "deliveredTokens": 3890, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.428571, + "recallAt10": 0.714286, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.593986, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.714286 + } + }, + "qarinahV2": { + "deliveredTokens": 3888, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.571429, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.599984, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.571429 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3888, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.571429, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.599984, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.571429 + } + }, + "oracle": { + "deliveredTokens": 3009, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7599, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.714286, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.688113, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.714286 + } + }, + "balancedV1": { + "deliveredTokens": 7599, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.428571, + "recallAt10": 0.714286, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.5, + "ndcgAt10": 0.593986, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.714286 + } + }, + "qarinahV2": { + "deliveredTokens": 7599, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.714286, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.688113, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.714286 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7599, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.714286, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.688113, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.714286 + } + }, + "oracle": { + "deliveredTokens": 3009, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.433864, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "scikit-learn/scikit-learn", + "instanceId": "scikit-learn__scikit-learn-25570", + "repositorySequence": 21, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 6212, + "balancedV1": 5992, + "qarinahV2": 6212, + "qarinahV2NoGraph": 6212, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 486, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 482, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 486, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 486, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 957, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 984, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 957, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 957, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1986, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1965, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1986, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1986, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3967, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3963, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3967, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3967, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7927, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 7992, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 7927, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7927, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.567316, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "scikit-learn/scikit-learn", + "instanceId": "scikit-learn__scikit-learn-25638", + "repositorySequence": 22, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.193426, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 7029, + "balancedV1": 5988, + "qarinahV2": 7029, + "qarinahV2NoGraph": 7029, + "oracle": 569 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 405, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 151, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 993, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 966, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 993, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 993, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 569, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1977, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 1904, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1977, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1977, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 569, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3901, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 3874, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3901, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3901, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 569, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 7767, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.193426, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 7991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 569, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.462054, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "scikit-learn/scikit-learn", + "instanceId": "scikit-learn__scikit-learn-25747", + "repositorySequence": 23, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.135652, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 7255, + "balancedV1": 7064, + "qarinahV2": 7255, + "qarinahV2NoGraph": 7255, + "oracle": 1027 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 475, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 475, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 475, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 151, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 961, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.234639, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 961, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 961, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 569, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1936, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1904, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1936, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1936, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 1027, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3995, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.181542, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 3901, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3995, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.181542, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3995, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.181542, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 1027, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 7911, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.135652, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 7997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 1027, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "DIRECTLY_SUPPORTED", + "decision": "ACCEPT_DIRECT", + "score": 0.658932, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "CONSERVATIVE_DIRECT_THRESHOLD_MET" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sphinx-doc/sphinx", + "instanceId": "sphinx-doc__sphinx-8273", + "repositorySequence": 4, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 921, + "balancedV1": 921, + "qarinahV2": 921, + "qarinahV2NoGraph": 921, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 418, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 418, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 418, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 418, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.202326, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "NO_CODE_ENTITY_MATCH", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 9, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sphinx-doc/sphinx", + "instanceId": "sphinx-doc__sphinx-8282", + "repositorySequence": 5, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1233, + "balancedV1": 1233, + "qarinahV2": 1233, + "qarinahV2NoGraph": 1233, + "oracle": 743 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 325, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 908, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 921, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 743, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1233, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1233, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1233, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1233, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 743, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1233, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1233, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1233, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1233, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 743, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1233, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1233, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1233, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1233, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 743, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "DIRECTLY_SUPPORTED", + "decision": "ACCEPT_DIRECT", + "score": 0.67521, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "CONSERVATIVE_DIRECT_THRESHOLD_MET" + ] + }, + "noTemporalFutureItems": 8, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sphinx-doc/sphinx", + "instanceId": "sphinx-doc__sphinx-8435", + "repositorySequence": 6, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.940915, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1505, + "balancedV1": 1505, + "qarinahV2": 1505, + "qarinahV2NoGraph": 1505, + "oracle": 1015 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 450, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.726229, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 450, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.726229, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 450, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.726229, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 450, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.726229, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 272, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.726229, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 868, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 909, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.847267, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 868, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 868, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 597, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1505, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1505, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.940915, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1505, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1505, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1015, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1505, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1505, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.940915, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1505, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1505, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1015, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1505, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1505, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.940915, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1505, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1505, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1015, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "DIRECTLY_SUPPORTED", + "decision": "ACCEPT_DIRECT", + "score": 0.742245, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "CONSERVATIVE_DIRECT_THRESHOLD_MET" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sphinx-doc/sphinx", + "instanceId": "sphinx-doc__sphinx-8474", + "repositorySequence": 7, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 1784, + "balancedV1": 1784, + "qarinahV2": 1784, + "qarinahV2NoGraph": 1784, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 490, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 450, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 490, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 490, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 815, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 863, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 815, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 815, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1784, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1784, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1784, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1784, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1784, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1784, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1784, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1784, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1784, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1784, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1784, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1784, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.3625, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sphinx-doc/sphinx", + "instanceId": "sphinx-doc__sphinx-8506", + "repositorySequence": 8, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1987, + "balancedV1": 1987, + "qarinahV2": 1987, + "qarinahV2NoGraph": 1987, + "oracle": 203 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 381, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 381, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 381, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 203, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 840, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 932, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 840, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 840, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 203, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1987, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1987, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1987, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1987, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 203, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 1987, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1987, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1987, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1987, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 203, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 1987, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1987, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1987, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1987, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 203, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.35125, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sphinx-doc/sphinx", + "instanceId": "sphinx-doc__sphinx-8595", + "repositorySequence": 9, + "positiveUnderStructuralOracle": true, + "directRecords": 2, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.848434, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2342, + "balancedV1": 2342, + "qarinahV2": 2342, + "qarinahV2NoGraph": 2342, + "oracle": 1294 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 450, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.515157, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.515157, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2": { + "deliveredTokens": 450, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.515157, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 450, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.515157, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 272, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.515157, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.25 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 969, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.926045, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + }, + "balancedV1": { + "deliveredTokens": 837, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.515157, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2": { + "deliveredTokens": 969, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.926045, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 969, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.926045, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + }, + "oracle": { + "deliveredTokens": 876, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.926045, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1827, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1924, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.794263, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2": { + "deliveredTokens": 1827, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1827, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1294, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2342, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2342, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.848434, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2342, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2342, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1294, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2342, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2342, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.848434, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2342, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2342, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1294, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "DIRECTLY_SUPPORTED", + "decision": "ACCEPT_DIRECT", + "score": 0.797273, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "CONSERVATIVE_DIRECT_THRESHOLD_MET" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sphinx-doc/sphinx", + "instanceId": "sphinx-doc__sphinx-8627", + "repositorySequence": 10, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 2567, + "balancedV1": 2567, + "qarinahV2": 2567, + "qarinahV2NoGraph": 2567, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 503, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 475, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 503, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 503, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 876, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 830, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 876, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 876, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1874, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1971, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1874, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1874, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2567, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 2567, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 2567, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2567, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2567, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 2567, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 2567, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2567, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.493088, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sphinx-doc/sphinx", + "instanceId": "sphinx-doc__sphinx-8713", + "repositorySequence": 11, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 4, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.2, + "recallAt5": 0.6, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.902569, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.2, + "recallAt5": 0.6, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.630208, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.2, + "recallAt5": 0.6, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.902569, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.2, + "recallAt5": 0.6, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.902569, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2963, + "balancedV1": 2963, + "qarinahV2": 2963, + "qarinahV2NoGraph": 2963, + "oracle": 1519 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 503, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.606249, + "directRecallAt10": 1, + "supportingRecallAt10": 0.2 + } + }, + "balancedV1": { + "deliveredTokens": 475, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.202083, + "directRecallAt10": 0, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2": { + "deliveredTokens": 503, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.606249, + "directRecallAt10": 1, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 503, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.606249, + "directRecallAt10": 1, + "supportingRecallAt10": 0.2 + } + }, + "oracle": { + "deliveredTokens": 325, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.606249, + "directRecallAt10": 1, + "supportingRecallAt10": 0.2 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 993, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.707291, + "directRecallAt10": 1, + "supportingRecallAt10": 0.4 + } + }, + "balancedV1": { + "deliveredTokens": 830, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.202083, + "directRecallAt10": 0, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2": { + "deliveredTokens": 993, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.707291, + "directRecallAt10": 1, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 993, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.707291, + "directRecallAt10": 1, + "supportingRecallAt10": 0.4 + } + }, + "oracle": { + "deliveredTokens": 968, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.6, + "recallAt10": 0.6, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.834791, + "directRecallAt10": 1, + "supportingRecallAt10": 0.6 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1948, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.6, + "recallAt10": 0.8, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.843442, + "directRecallAt10": 1, + "supportingRecallAt10": 0.8 + } + }, + "balancedV1": { + "deliveredTokens": 1971, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.6, + "recallAt10": 0.8, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.569375, + "directRecallAt10": 1, + "supportingRecallAt10": 0.8 + } + }, + "qarinahV2": { + "deliveredTokens": 1948, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.6, + "recallAt10": 0.8, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.843442, + "directRecallAt10": 1, + "supportingRecallAt10": 0.8 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1948, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.6, + "recallAt10": 0.8, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.843442, + "directRecallAt10": 1, + "supportingRecallAt10": 0.8 + } + }, + "oracle": { + "deliveredTokens": 1519, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 2963, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.6, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.902569, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2963, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.6, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.630208, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2963, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.6, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.902569, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2963, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.6, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.902569, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1519, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 2963, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.6, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.902569, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 2963, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.6, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.630208, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 2963, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.6, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.902569, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2963, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.6, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.902569, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 1519, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.3999, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sphinx-doc/sphinx", + "instanceId": "sphinx-doc__sphinx-8721", + "repositorySequence": 12, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 6, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.993078, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.971476, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.993078, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.993078, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3136, + "balancedV1": 3313, + "qarinahV2": 3136, + "qarinahV2NoGraph": 3136, + "oracle": 2047 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.302602, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "balancedV1": { + "deliveredTokens": 504, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.493523, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.302602, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 418, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.302602, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "oracle": { + "deliveredTokens": 325, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.302602, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 922, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.644824, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 956, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.493523, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 922, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.644824, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 922, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.644824, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 968, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.644824, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1915, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 0.833333, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.892211, + "directRecallAt10": null, + "supportingRecallAt10": 0.833333 + } + }, + "balancedV1": { + "deliveredTokens": 1915, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.775148, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 1915, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 0.833333, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.892211, + "directRecallAt10": null, + "supportingRecallAt10": 0.833333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1915, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 0.833333, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.892211, + "directRecallAt10": null, + "supportingRecallAt10": 0.833333 + } + }, + "oracle": { + "deliveredTokens": 1519, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 0.833333, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.892211, + "directRecallAt10": null, + "supportingRecallAt10": 0.833333 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3491, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.993078, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3491, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.971476, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3491, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.993078, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3491, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.993078, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2047, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3491, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.993078, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3491, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.971476, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3491, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.993078, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3491, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.993078, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2047, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "DIRECTLY_SUPPORTED", + "decision": "ACCEPT_DIRECT", + "score": 0.694079, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "CONSERVATIVE_DIRECT_THRESHOLD_MET" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sphinx-doc/sphinx", + "instanceId": "sphinx-doc__sphinx-8801", + "repositorySequence": 13, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 7, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.142857, + "recallAt5": 0.571429, + "recallAt10": 0.857143, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.884784, + "directRecallAt10": null, + "supportingRecallAt10": 0.857143 + }, + "qarinahV2": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3314, + "balancedV1": 3313, + "qarinahV2": 3314, + "qarinahV2NoGraph": 3314, + "oracle": 2251 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 497, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.448304, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + }, + "balancedV1": { + "deliveredTokens": 504, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.448304, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + }, + "qarinahV2": { + "deliveredTokens": 497, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.448304, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 497, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.448304, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + }, + "oracle": { + "deliveredTokens": 325, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.274876, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 980, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.571429, + "recallAt10": 0.571429, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.704125, + "directRecallAt10": null, + "supportingRecallAt10": 0.571429 + } + }, + "balancedV1": { + "deliveredTokens": 956, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.448304, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + }, + "qarinahV2": { + "deliveredTokens": 980, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.571429, + "recallAt10": 0.571429, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.704125, + "directRecallAt10": null, + "supportingRecallAt10": 0.571429 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 980, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.571429, + "recallAt10": 0.571429, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.704125, + "directRecallAt10": null, + "supportingRecallAt10": 0.571429 + } + }, + "oracle": { + "deliveredTokens": 968, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.428571, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.585742, + "directRecallAt10": null, + "supportingRecallAt10": 0.428571 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1901, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 0.857143, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.908375, + "directRecallAt10": null, + "supportingRecallAt10": 0.857143 + } + }, + "balancedV1": { + "deliveredTokens": 1984, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.571429, + "recallAt10": 0.714286, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.802038, + "directRecallAt10": null, + "supportingRecallAt10": 0.714286 + } + }, + "qarinahV2": { + "deliveredTokens": 1901, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 0.857143, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.908375, + "directRecallAt10": null, + "supportingRecallAt10": 0.857143 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1901, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 0.857143, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.908375, + "directRecallAt10": null, + "supportingRecallAt10": 0.857143 + } + }, + "oracle": { + "deliveredTokens": 1723, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 0.857143, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.908375, + "directRecallAt10": null, + "supportingRecallAt10": 0.857143 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3695, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3695, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.571429, + "recallAt10": 0.857143, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.884784, + "directRecallAt10": null, + "supportingRecallAt10": 0.857143 + } + }, + "qarinahV2": { + "deliveredTokens": 3695, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3695, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2251, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3695, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3695, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.571429, + "recallAt10": 0.857143, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.884784, + "directRecallAt10": null, + "supportingRecallAt10": 0.857143 + } + }, + "qarinahV2": { + "deliveredTokens": 3695, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3695, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 2251, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "DIRECTLY_SUPPORTED", + "decision": "ACCEPT_DIRECT", + "score": 0.7, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "CONSERVATIVE_DIRECT_THRESHOLD_MET" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sphinx-doc/sphinx", + "instanceId": "sphinx-doc__sphinx-10325", + "repositorySequence": 14, + "positiveUnderStructuralOracle": true, + "directRecords": 3, + "supportingRecords": 5, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.625, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.301758, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.625 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.375, + "recallAt10": 0.625, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.506353, + "directRecallAt10": 1, + "supportingRecallAt10": 0.625 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.625, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.301758, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.625 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.625, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.301758, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.625 + }, + "oracle": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3228, + "balancedV1": 3073, + "qarinahV2": 3228, + "qarinahV2NoGraph": 3228, + "oracle": 2539 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 466, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.076799, + "directRecallAt10": 0, + "supportingRecallAt10": 0.125 + } + }, + "balancedV1": { + "deliveredTokens": 355, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 466, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.076799, + "directRecallAt10": 0, + "supportingRecallAt10": 0.125 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 466, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.076799, + "directRecallAt10": 0, + "supportingRecallAt10": 0.125 + } + }, + "oracle": { + "deliveredTokens": 497, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.595569, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.25 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 821, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.060862, + "directRecallAt10": 0, + "supportingRecallAt10": 0.125 + } + }, + "balancedV1": { + "deliveredTokens": 883, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.060862, + "directRecallAt10": 0, + "supportingRecallAt10": 0.125 + } + }, + "qarinahV2": { + "deliveredTokens": 821, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.060862, + "directRecallAt10": 0, + "supportingRecallAt10": 0.125 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 821, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.060862, + "directRecallAt10": 0, + "supportingRecallAt10": 0.125 + } + }, + "oracle": { + "deliveredTokens": 980, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.830578, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1864, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.113285, + "directRecallAt10": 0, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 1915, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.375, + "recallAt10": 0.5, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.38463, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 1864, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.113285, + "directRecallAt10": 0, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1864, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.113285, + "directRecallAt10": 0, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 1807, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 0.75, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.921026, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3983, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.625, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.301758, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.625 + } + }, + "balancedV1": { + "deliveredTokens": 3983, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.375, + "recallAt10": 0.625, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.506353, + "directRecallAt10": 1, + "supportingRecallAt10": 0.625 + } + }, + "qarinahV2": { + "deliveredTokens": 3983, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.625, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.301758, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.625 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3983, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.625, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.301758, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.625 + } + }, + "oracle": { + "deliveredTokens": 2539, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3983, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.625, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.301758, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.625 + } + }, + "balancedV1": { + "deliveredTokens": 3983, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.375, + "recallAt10": 0.625, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.506353, + "directRecallAt10": 1, + "supportingRecallAt10": 0.625 + } + }, + "qarinahV2": { + "deliveredTokens": 3983, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.625, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.301758, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.625 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3983, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.625, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.301758, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.625 + } + }, + "oracle": { + "deliveredTokens": 2539, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.316667, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sphinx-doc/sphinx", + "instanceId": "sphinx-doc__sphinx-10451", + "repositorySequence": 15, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 9, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.111111, + "recallAt5": 0.444444, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.737573, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "balancedV1": { + "recallAt1": 0.111111, + "recallAt5": 0.444444, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.754591, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2": { + "recallAt1": 0.111111, + "recallAt5": 0.444444, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.737573, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.111111, + "recallAt5": 0.444444, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.737573, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "oracle": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3230, + "balancedV1": 3246, + "qarinahV2": 3230, + "qarinahV2NoGraph": 3230, + "oracle": 2890 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 497, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.383343, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + }, + "balancedV1": { + "deliveredTokens": 503, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.235046, + "directRecallAt10": null, + "supportingRecallAt10": 0.111111 + } + }, + "qarinahV2": { + "deliveredTokens": 497, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.383343, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 497, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.383343, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + }, + "oracle": { + "deliveredTokens": 325, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.235046, + "directRecallAt10": null, + "supportingRecallAt10": 0.111111 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 978, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.383343, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + }, + "balancedV1": { + "deliveredTokens": 853, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.383343, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + }, + "qarinahV2": { + "deliveredTokens": 978, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.383343, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 978, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.383343, + "directRecallAt10": null, + "supportingRecallAt10": 0.222222 + } + }, + "oracle": { + "deliveredTokens": 968, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.500866, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1872, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.444444, + "recallAt10": 0.444444, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.575499, + "directRecallAt10": null, + "supportingRecallAt10": 0.444444 + } + }, + "balancedV1": { + "deliveredTokens": 1831, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.444444, + "recallAt10": 0.444444, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.602094, + "directRecallAt10": null, + "supportingRecallAt10": 0.444444 + } + }, + "qarinahV2": { + "deliveredTokens": 1872, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.444444, + "recallAt10": 0.444444, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.575499, + "directRecallAt10": null, + "supportingRecallAt10": 0.444444 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1872, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.444444, + "recallAt10": 0.444444, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.575499, + "directRecallAt10": null, + "supportingRecallAt10": 0.444444 + } + }, + "oracle": { + "deliveredTokens": 1870, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 0.666667, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.776747, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3927, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.444444, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.737573, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 3916, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.444444, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.754591, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3927, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.444444, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.737573, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3927, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.444444, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.737573, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 2890, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 4334, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.444444, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.737573, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 4334, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.444444, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.754591, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 4334, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.444444, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.737573, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 4334, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.444444, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.737573, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 2890, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.4, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "NO_CODE_ENTITY_MATCH", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sphinx-doc/sphinx", + "instanceId": "sphinx-doc__sphinx-11445", + "repositorySequence": 16, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.30103, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3012, + "balancedV1": 3352, + "qarinahV2": 3012, + "qarinahV2NoGraph": 3012, + "oracle": 396 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 396, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 503, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 396, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 396, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 396, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 992, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 879, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 992, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 992, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 396, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1872, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1987, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1872, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1872, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 396, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3806, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3842, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.30103, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3806, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3806, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 396, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 4816, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 4816, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.111111, + "ndcgAt10": 0.30103, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 4816, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 4816, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 396, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.425723, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-13647", + "repositorySequence": 16, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.618289, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.732829, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.618289, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.618289, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2579, + "balancedV1": 2672, + "qarinahV2": 2579, + "qarinahV2NoGraph": 2579, + "oracle": 948 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 424, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 488, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.296082, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 424, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 424, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 365, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 904, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.436747, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 975, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.530721, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 904, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.436747, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 904, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.436747, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 948, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1937, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.618289, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1985, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.732829, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1937, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.618289, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1937, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.618289, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 948, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.618289, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.732829, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.618289, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.618289, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 948, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.618289, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.732829, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.618289, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3433, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.618289, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 948, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.310811, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "NO_CODE_ENTITY_MATCH", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 8, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-13773", + "repositorySequence": 17, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.545233, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.47082, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.545233, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.545233, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + }, + "oracle": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2301, + "balancedV1": 2326, + "qarinahV2": 2301, + "qarinahV2NoGraph": 2301, + "oracle": 1274 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 466, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 466, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 466, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 466, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 326, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.657663, + "directRecallAt10": 1, + "supportingRecallAt10": 0.25 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 942, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.467145, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 990, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138313, + "directRecallAt10": 0, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2": { + "deliveredTokens": 942, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.467145, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 942, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.467145, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 928, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.905587, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1908, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.545233, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + }, + "balancedV1": { + "deliveredTokens": 1936, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.47082, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2": { + "deliveredTokens": 1908, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.545233, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1908, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.545233, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + }, + "oracle": { + "deliveredTokens": 1274, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3759, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.545233, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + }, + "balancedV1": { + "deliveredTokens": 3759, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.47082, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2": { + "deliveredTokens": 3759, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.545233, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3759, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.545233, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + }, + "oracle": { + "deliveredTokens": 1274, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3759, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.545233, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + }, + "balancedV1": { + "deliveredTokens": 3759, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.47082, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2": { + "deliveredTokens": 3759, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.545233, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3759, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.545233, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + }, + "oracle": { + "deliveredTokens": 1274, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.499857, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 10, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-13895", + "repositorySequence": 18, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.433677, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.25, + "ndcgAt10": 0.406228, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.433677, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.433677, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2618, + "balancedV1": 2524, + "qarinahV2": 2618, + "qarinahV2NoGraph": 2618, + "oracle": 516 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 502, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.152733, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 502, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.152733, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 502, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.152733, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 373, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 952, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.25699, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 979, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.25, + "ndcgAt10": 0.197904, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 952, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.25699, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 952, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.25699, + "directRecallAt10": 0, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 516, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1945, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.433677, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 1915, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.104257, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 1945, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.433677, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1945, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.433677, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 516, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3980, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.433677, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 3980, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.25, + "ndcgAt10": 0.406228, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3980, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.433677, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3980, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.433677, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 516, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3980, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.433677, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 3980, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.25, + "ndcgAt10": 0.406228, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3980, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.433677, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3980, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.433677, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 516, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "DIRECTLY_SUPPORTED", + "decision": "ACCEPT_DIRECT", + "score": 0.71875, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "CONSERVATIVE_DIRECT_THRESHOLD_MET" + ] + }, + "noTemporalFutureItems": 8, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-13915", + "repositorySequence": 19, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 4, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.476163, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + }, + "balancedV1": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.58557, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.476163, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.476163, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + }, + "oracle": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2390, + "balancedV1": 2621, + "qarinahV2": 2390, + "qarinahV2NoGraph": 2390, + "oracle": 708 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 407, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.246302, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 443, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 407, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.246302, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 407, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.246302, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 373, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 927, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.19519, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 990, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.58557, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 927, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.19519, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 927, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.19519, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 708, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1921, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.363318, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 1938, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.58557, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 1921, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.363318, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1921, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.363318, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 708, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3949, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.476163, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "balancedV1": { + "deliveredTokens": 3935, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.58557, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 3949, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.476163, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3949, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.476163, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "oracle": { + "deliveredTokens": 708, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 4172, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.476163, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "balancedV1": { + "deliveredTokens": 4172, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.58557, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 4172, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.476163, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 4172, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.476163, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "oracle": { + "deliveredTokens": 708, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "DIRECTLY_SUPPORTED", + "decision": "ACCEPT_DIRECT", + "score": 0.678571, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "CONSERVATIVE_DIRECT_THRESHOLD_MET" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-13971", + "repositorySequence": 20, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.433808, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.666667, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.314948, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.433808, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.433808, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2447, + "balancedV1": 2316, + "qarinahV2": 2447, + "qarinahV2NoGraph": 2447, + "oracle": 578 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 436, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 472, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 436, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 436, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 355, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.433808, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 990, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.433808, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.433808, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1927, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.433808, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 1934, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.333333, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.242076, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 1927, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.433808, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1927, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.433808, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3872, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.433808, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 3872, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.666667, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.314948, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3872, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.433808, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3872, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.433808, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 4397, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.433808, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 4397, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.666667, + "precisionAt5": 0, + "precisionAt10": 0.2, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.314948, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 4397, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.433808, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 4397, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.433808, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 578, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.45, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 8, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-14024", + "repositorySequence": 21, + "positiveUnderStructuralOracle": true, + "directRecords": 2, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.4, + "recallAt10": 0.6, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.396801, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.6 + }, + "balancedV1": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.8, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.791333, + "directRecallAt10": 1, + "supportingRecallAt10": 0.8 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.4, + "recallAt10": 0.6, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.396801, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.6 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.4, + "recallAt10": 0.6, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.396801, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.6 + }, + "oracle": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2182, + "balancedV1": 1992, + "qarinahV2": 2182, + "qarinahV2NoGraph": 2182, + "oracle": 933 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 468, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.101594, + "directRecallAt10": 0, + "supportingRecallAt10": 0.2 + } + }, + "balancedV1": { + "deliveredTokens": 405, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.483067, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2": { + "deliveredTokens": 468, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.101594, + "directRecallAt10": 0, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 468, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.101594, + "directRecallAt10": 0, + "supportingRecallAt10": 0.2 + } + }, + "oracle": { + "deliveredTokens": 443, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.787848, + "directRecallAt10": 1, + "supportingRecallAt10": 0.4 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 987, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.343127, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.4 + } + }, + "balancedV1": { + "deliveredTokens": 955, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.6, + "recallAt10": 0.6, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.753405, + "directRecallAt10": 1, + "supportingRecallAt10": 0.6 + } + }, + "qarinahV2": { + "deliveredTokens": 987, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.343127, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 987, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.343127, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.4 + } + }, + "oracle": { + "deliveredTokens": 933, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1863, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.4, + "recallAt10": 0.6, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.396801, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.6 + } + }, + "balancedV1": { + "deliveredTokens": 1992, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.8, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.791333, + "directRecallAt10": 1, + "supportingRecallAt10": 0.8 + } + }, + "qarinahV2": { + "deliveredTokens": 1863, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.4, + "recallAt10": 0.6, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.396801, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.6 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1863, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.4, + "recallAt10": 0.6, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.396801, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.6 + } + }, + "oracle": { + "deliveredTokens": 933, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3414, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.4, + "recallAt10": 0.6, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.396801, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.6 + } + }, + "balancedV1": { + "deliveredTokens": 3963, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.8, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.791333, + "directRecallAt10": 1, + "supportingRecallAt10": 0.8 + } + }, + "qarinahV2": { + "deliveredTokens": 3874, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.4, + "recallAt10": 0.6, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.396801, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.6 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3874, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.4, + "recallAt10": 0.6, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.396801, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.6 + } + }, + "oracle": { + "deliveredTokens": 933, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3414, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.4, + "recallAt10": 0.6, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.396801, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.6 + } + }, + "balancedV1": { + "deliveredTokens": 4574, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.8, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.791333, + "directRecallAt10": 1, + "supportingRecallAt10": 0.8 + } + }, + "qarinahV2": { + "deliveredTokens": 4574, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.4, + "recallAt10": 0.6, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.396801, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.6 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 4574, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.4, + "recallAt10": 0.6, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.396801, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.6 + } + }, + "oracle": { + "deliveredTokens": 933, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.41875, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "NO_CODE_ENTITY_MATCH", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-14308", + "repositorySequence": 22, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 4, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.905218, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.759834, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + }, + "qarinahV2": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.905218, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.905218, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2613, + "balancedV1": 2568, + "qarinahV2": 2613, + "qarinahV2NoGraph": 2613, + "oracle": 755 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 436, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 512, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 436, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 436, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 365, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 956, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 967, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 956, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 956, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 755, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1904, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.787702, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "balancedV1": { + "deliveredTokens": 1975, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.87735, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 1904, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.787702, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1904, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.787702, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "oracle": { + "deliveredTokens": 755, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3878, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.905218, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3875, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.759834, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2": { + "deliveredTokens": 3878, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.905218, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3878, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.905218, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 755, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 4804, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.905218, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 4804, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.759834, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2": { + "deliveredTokens": 4804, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.905218, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 4804, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.905218, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 755, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.536071, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-14317", + "repositorySequence": 23, + "positiveUnderStructuralOracle": true, + "directRecords": 2, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.2, + "recallAt5": 0.8, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.924014, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.8, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.893678, + "directRecallAt10": 1, + "supportingRecallAt10": 0.8 + }, + "qarinahV2": { + "recallAt1": 0.2, + "recallAt5": 0.8, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.924014, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.2, + "recallAt5": 0.8, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.924014, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 1800, + "balancedV1": 1984, + "qarinahV2": 1800, + "qarinahV2NoGraph": 1800, + "oracle": 877 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 436, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.584661, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.4 + } + }, + "balancedV1": { + "deliveredTokens": 512, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.6, + "recallAt10": 0.6, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.86836, + "directRecallAt10": 1, + "supportingRecallAt10": 0.6 + } + }, + "qarinahV2": { + "deliveredTokens": 436, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.584661, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 436, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.584661, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.4 + } + }, + "oracle": { + "deliveredTokens": 512, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.6, + "recallAt10": 0.6, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.86836, + "directRecallAt10": 1, + "supportingRecallAt10": 0.6 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 905, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.8, + "recallAt10": 0.8, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.873218, + "directRecallAt10": 1, + "supportingRecallAt10": 0.8 + } + }, + "balancedV1": { + "deliveredTokens": 984, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.787848, + "directRecallAt10": 1, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2": { + "deliveredTokens": 905, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.8, + "recallAt10": 0.8, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.873218, + "directRecallAt10": 1, + "supportingRecallAt10": 0.8 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 905, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.8, + "recallAt10": 0.8, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.873218, + "directRecallAt10": 1, + "supportingRecallAt10": 0.8 + } + }, + "oracle": { + "deliveredTokens": 877, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1967, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.8, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.924014, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1984, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.8, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.893678, + "directRecallAt10": 1, + "supportingRecallAt10": 0.8 + } + }, + "qarinahV2": { + "deliveredTokens": 1967, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.8, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.924014, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1967, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.8, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.924014, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 877, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3836, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.8, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.924014, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3958, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.8, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.893678, + "directRecallAt10": 1, + "supportingRecallAt10": 0.8 + } + }, + "qarinahV2": { + "deliveredTokens": 3958, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.8, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.924014, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3958, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.8, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.924014, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 877, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 3836, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.8, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.924014, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 4926, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.8, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.893678, + "directRecallAt10": 1, + "supportingRecallAt10": 0.8 + } + }, + "qarinahV2": { + "deliveredTokens": 4926, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.8, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.924014, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 4926, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.8, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.924014, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 877, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.55, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-14396", + "repositorySequence": 24, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.315465, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.289065, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.315465, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.315465, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2589, + "balancedV1": 2709, + "qarinahV2": 2589, + "qarinahV2NoGraph": 2589, + "oracle": 520 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 472, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 472, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 472, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 472, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 0, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 979, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 899, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 979, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 979, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 520, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1987, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1983, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1987, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1987, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 520, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3974, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.315465, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3964, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.289065, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 3974, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.315465, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3974, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.315465, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 520, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 5140, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.315465, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 5140, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.289065, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2": { + "deliveredTokens": 5140, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.315465, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 5140, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 1, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.315465, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 520, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.343643, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 10, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-14774", + "repositorySequence": 25, + "positiveUnderStructuralOracle": true, + "directRecords": 3, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 0.833333, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.887068, + "directRecallAt10": 1, + "supportingRecallAt10": 0.833333 + }, + "balancedV1": { + "recallAt1": 0.166667, + "recallAt5": 0.5, + "recallAt10": 0.666667, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.884662, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 0.833333, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.887068, + "directRecallAt10": 1, + "supportingRecallAt10": 0.833333 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 0.833333, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.887068, + "directRecallAt10": 1, + "supportingRecallAt10": 0.833333 + }, + "oracle": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2041, + "balancedV1": 2262, + "qarinahV2": 2041, + "qarinahV2NoGraph": 2041, + "oracle": 1091 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 437, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.479867, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 391, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.646636, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 437, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.479867, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 437, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.479867, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 391, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.646636, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.333333 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 997, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.848865, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 995, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.844878, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 997, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.848865, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 997, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.848865, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 969, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 0.833333, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.952923, + "directRecallAt10": 1, + "supportingRecallAt10": 0.833333 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1919, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.848865, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 1994, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.844878, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 1919, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.848865, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1919, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.848865, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 1091, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3982, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 0.833333, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.887068, + "directRecallAt10": 1, + "supportingRecallAt10": 0.833333 + } + }, + "balancedV1": { + "deliveredTokens": 3991, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.5, + "recallAt10": 0.666667, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.884662, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3982, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 0.833333, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.887068, + "directRecallAt10": 1, + "supportingRecallAt10": 0.833333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3982, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 0.833333, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.887068, + "directRecallAt10": 1, + "supportingRecallAt10": 0.833333 + } + }, + "oracle": { + "deliveredTokens": 1091, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 5712, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 0.833333, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.887068, + "directRecallAt10": 1, + "supportingRecallAt10": 0.833333 + } + }, + "balancedV1": { + "deliveredTokens": 5712, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.5, + "recallAt10": 0.666667, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.884662, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 5712, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 0.833333, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.887068, + "directRecallAt10": 1, + "supportingRecallAt10": 0.833333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 5712, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 0.833333, + "precisionAt5": 0.8, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.887068, + "directRecallAt10": 1, + "supportingRecallAt10": 0.833333 + } + }, + "oracle": { + "deliveredTokens": 1091, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.49693, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-14817", + "repositorySequence": 26, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 6, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.714286, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.520613, + "directRecallAt10": 1, + "supportingRecallAt10": 0.714286 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.428571, + "recallAt10": 0.571429, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.296867, + "directRecallAt10": 0, + "supportingRecallAt10": 0.571429 + }, + "qarinahV2": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.714286, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.520613, + "directRecallAt10": 1, + "supportingRecallAt10": 0.714286 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.714286, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.520613, + "directRecallAt10": 1, + "supportingRecallAt10": 0.714286 + }, + "oracle": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2524, + "balancedV1": 2389, + "qarinahV2": 2524, + "qarinahV2NoGraph": 2524, + "oracle": 1291 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 444, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.177368, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + } + }, + "balancedV1": { + "deliveredTokens": 472, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 444, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.177368, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 444, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.177368, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + } + }, + "oracle": { + "deliveredTokens": 478, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.428571, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.732694, + "directRecallAt10": 1, + "supportingRecallAt10": 0.428571 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 952, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.177368, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + } + }, + "balancedV1": { + "deliveredTokens": 899, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.165072, + "directRecallAt10": 0, + "supportingRecallAt10": 0.285714 + } + }, + "qarinahV2": { + "deliveredTokens": 952, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.177368, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 952, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.177368, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + } + }, + "oracle": { + "deliveredTokens": 878, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 0.714286, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.877698, + "directRecallAt10": 1, + "supportingRecallAt10": 0.714286 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1952, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.714286, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.530416, + "directRecallAt10": 1, + "supportingRecallAt10": 0.714286 + } + }, + "balancedV1": { + "deliveredTokens": 1964, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.428571, + "recallAt10": 0.571429, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.296867, + "directRecallAt10": 0, + "supportingRecallAt10": 0.571429 + } + }, + "qarinahV2": { + "deliveredTokens": 1952, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.714286, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.530416, + "directRecallAt10": 1, + "supportingRecallAt10": 0.714286 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1952, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.714286, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.530416, + "directRecallAt10": 1, + "supportingRecallAt10": 0.714286 + } + }, + "oracle": { + "deliveredTokens": 1291, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3928, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.714286, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.520613, + "directRecallAt10": 1, + "supportingRecallAt10": 0.714286 + } + }, + "balancedV1": { + "deliveredTokens": 3965, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.428571, + "recallAt10": 0.571429, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.296867, + "directRecallAt10": 0, + "supportingRecallAt10": 0.571429 + } + }, + "qarinahV2": { + "deliveredTokens": 3928, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.714286, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.520613, + "directRecallAt10": 1, + "supportingRecallAt10": 0.714286 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3928, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.714286, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.520613, + "directRecallAt10": 1, + "supportingRecallAt10": 0.714286 + } + }, + "oracle": { + "deliveredTokens": 1291, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 5912, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.714286, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.520613, + "directRecallAt10": 1, + "supportingRecallAt10": 0.714286 + } + }, + "balancedV1": { + "deliveredTokens": 5912, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.428571, + "recallAt10": 0.571429, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.296867, + "directRecallAt10": 0, + "supportingRecallAt10": 0.571429 + } + }, + "qarinahV2": { + "deliveredTokens": 5912, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.714286, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.520613, + "directRecallAt10": 1, + "supportingRecallAt10": 0.714286 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 5912, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.714286, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.520613, + "directRecallAt10": 1, + "supportingRecallAt10": 0.714286 + } + }, + "oracle": { + "deliveredTokens": 1291, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.391422, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 10, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-15011", + "repositorySequence": 27, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 2421, + "balancedV1": 2507, + "qarinahV2": 2421, + "qarinahV2NoGraph": 2421, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 423, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 451, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 423, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 423, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 976, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 974, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 976, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 976, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1933, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1942, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1933, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1933, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3985, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3887, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3985, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3985, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 6465, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 6465, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 6465, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 6465, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.461707, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-15308", + "repositorySequence": 28, + "positiveUnderStructuralOracle": true, + "directRecords": 4, + "supportingRecords": 4, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.125, + "recallAt5": 0.5, + "recallAt10": 0.875, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.844205, + "directRecallAt10": 1, + "supportingRecallAt10": 0.875 + }, + "balancedV1": { + "recallAt1": 0.125, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.847606, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + }, + "qarinahV2": { + "recallAt1": 0.125, + "recallAt5": 0.5, + "recallAt10": 0.875, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.844205, + "directRecallAt10": 1, + "supportingRecallAt10": 0.875 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.125, + "recallAt5": 0.5, + "recallAt10": 0.875, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.844205, + "directRecallAt10": 1, + "supportingRecallAt10": 0.875 + }, + "oracle": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2531, + "balancedV1": 2469, + "qarinahV2": 2531, + "qarinahV2NoGraph": 2531, + "oracle": 1844 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 435, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.330517, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.125 + } + }, + "balancedV1": { + "deliveredTokens": 414, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.539051, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2": { + "deliveredTokens": 435, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.330517, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.125 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 435, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.330517, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.125 + } + }, + "oracle": { + "deliveredTokens": 391, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.539051, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.25 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 990, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.680743, + "directRecallAt10": 0.75, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 967, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.375, + "recallAt10": 0.375, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.565287, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.375 + } + }, + "qarinahV2": { + "deliveredTokens": 990, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.680743, + "directRecallAt10": 0.75, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 990, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.680743, + "directRecallAt10": 0.75, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 946, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 0.625, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.889276, + "directRecallAt10": 1, + "supportingRecallAt10": 0.625 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1989, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.680743, + "directRecallAt10": 0.75, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 1951, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.847606, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2": { + "deliveredTokens": 1989, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.680743, + "directRecallAt10": 0.75, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1989, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.680743, + "directRecallAt10": 0.75, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 1844, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3925, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.5, + "recallAt10": 0.875, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.844205, + "directRecallAt10": 1, + "supportingRecallAt10": 0.875 + } + }, + "balancedV1": { + "deliveredTokens": 3944, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.847606, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2": { + "deliveredTokens": 3925, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.5, + "recallAt10": 0.875, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.844205, + "directRecallAt10": 1, + "supportingRecallAt10": 0.875 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3925, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.5, + "recallAt10": 0.875, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.844205, + "directRecallAt10": 1, + "supportingRecallAt10": 0.875 + } + }, + "oracle": { + "deliveredTokens": 1844, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 6707, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.5, + "recallAt10": 0.875, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.844205, + "directRecallAt10": 1, + "supportingRecallAt10": 0.875 + } + }, + "balancedV1": { + "deliveredTokens": 6707, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.847606, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2": { + "deliveredTokens": 6707, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.5, + "recallAt10": 0.875, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.844205, + "directRecallAt10": 1, + "supportingRecallAt10": 0.875 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 6707, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.5, + "recallAt10": 0.875, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.844205, + "directRecallAt10": 1, + "supportingRecallAt10": 0.875 + } + }, + "oracle": { + "deliveredTokens": 1844, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.55, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-15345", + "repositorySequence": 29, + "positiveUnderStructuralOracle": true, + "directRecords": 2, + "supportingRecords": 7, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.615373, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + }, + "balancedV1": { + "recallAt1": 0.111111, + "recallAt5": 0.444444, + "recallAt10": 0.777778, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.627342, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.777778 + }, + "qarinahV2": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.615373, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.615373, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + }, + "oracle": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2628, + "balancedV1": 2418, + "qarinahV2": 2628, + "qarinahV2NoGraph": 2628, + "oracle": 1985 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 465, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39913, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.111111 + } + }, + "balancedV1": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.451388, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 465, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39913, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.111111 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 465, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.111111, + "recallAt10": 0.111111, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.39913, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.111111 + } + }, + "oracle": { + "deliveredTokens": 423, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.650952, + "directRecallAt10": 1, + "supportingRecallAt10": 0.222222 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 886, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.571025, + "directRecallAt10": 1, + "supportingRecallAt10": 0.222222 + } + }, + "balancedV1": { + "deliveredTokens": 952, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.444444, + "recallAt10": 0.444444, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.493633, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.444444 + } + }, + "qarinahV2": { + "deliveredTokens": 886, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.571025, + "directRecallAt10": 1, + "supportingRecallAt10": 0.222222 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 886, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.571025, + "directRecallAt10": 1, + "supportingRecallAt10": 0.222222 + } + }, + "oracle": { + "deliveredTokens": 956, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 0.555556, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.826241, + "directRecallAt10": 1, + "supportingRecallAt10": 0.555556 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1913, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.615373, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 1984, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.444444, + "recallAt10": 0.777778, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.627342, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.777778 + } + }, + "qarinahV2": { + "deliveredTokens": 1913, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.615373, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1913, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.615373, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 1985, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3891, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.615373, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 3955, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.444444, + "recallAt10": 0.777778, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.627342, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.777778 + } + }, + "qarinahV2": { + "deliveredTokens": 3891, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.615373, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3891, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.615373, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 1985, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 6848, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.615373, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 6848, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.444444, + "recallAt10": 0.777778, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.627342, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.777778 + } + }, + "qarinahV2": { + "deliveredTokens": 6848, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.615373, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 6848, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.222222, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.615373, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 1985, + "metrics": { + "recallAt1": 0.111111, + "recallAt5": 0.555556, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.592105, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 9, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-15346", + "repositorySequence": 30, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 2637, + "balancedV1": 2504, + "qarinahV2": 2637, + "qarinahV2NoGraph": 2637, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 468, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 481, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 468, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 468, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 988, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 948, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 988, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 988, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1973, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1951, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1973, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1973, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3975, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3894, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3975, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3975, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 6974, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 6974, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 6974, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 6974, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.45625, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 9, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-15609", + "repositorySequence": 31, + "positiveUnderStructuralOracle": true, + "directRecords": 5, + "supportingRecords": 5, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.1, + "recallAt5": 0.4, + "recallAt10": 0.7, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.87938, + "directRecallAt10": 1, + "supportingRecallAt10": 0.7 + }, + "balancedV1": { + "recallAt1": 0.1, + "recallAt5": 0.4, + "recallAt10": 0.8, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.912415, + "directRecallAt10": 1, + "supportingRecallAt10": 0.8 + }, + "qarinahV2": { + "recallAt1": 0.1, + "recallAt5": 0.4, + "recallAt10": 0.7, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.87938, + "directRecallAt10": 1, + "supportingRecallAt10": 0.7 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.1, + "recallAt5": 0.4, + "recallAt10": 0.7, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.87938, + "directRecallAt10": 1, + "supportingRecallAt10": 0.7 + }, + "oracle": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2439, + "balancedV1": 2290, + "qarinahV2": 2439, + "qarinahV2NoGraph": 2439, + "oracle": 2111 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 498, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.468637, + "directRecallAt10": 0.4, + "supportingRecallAt10": 0.2 + } + }, + "balancedV1": { + "deliveredTokens": 467, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.3, + "recallAt10": 0.3, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.516527, + "directRecallAt10": 0.4, + "supportingRecallAt10": 0.3 + } + }, + "qarinahV2": { + "deliveredTokens": 498, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.468637, + "directRecallAt10": 0.4, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 498, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.468637, + "directRecallAt10": 0.4, + "supportingRecallAt10": 0.2 + } + }, + "oracle": { + "deliveredTokens": 391, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.468637, + "directRecallAt10": 0.4, + "supportingRecallAt10": 0.2 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 983, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.723468, + "directRecallAt10": 0.8, + "supportingRecallAt10": 0.4 + } + }, + "balancedV1": { + "deliveredTokens": 983, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.723468, + "directRecallAt10": 0.8, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2": { + "deliveredTokens": 983, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.723468, + "directRecallAt10": 0.8, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 983, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.723468, + "directRecallAt10": 0.8, + "supportingRecallAt10": 0.4 + } + }, + "oracle": { + "deliveredTokens": 945, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.84722, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1991, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.4, + "recallAt10": 0.6, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.851694, + "directRecallAt10": 1, + "supportingRecallAt10": 0.6 + } + }, + "balancedV1": { + "deliveredTokens": 1997, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.4, + "recallAt10": 0.8, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.912415, + "directRecallAt10": 1, + "supportingRecallAt10": 0.8 + } + }, + "qarinahV2": { + "deliveredTokens": 1991, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.4, + "recallAt10": 0.6, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.851694, + "directRecallAt10": 1, + "supportingRecallAt10": 0.6 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1991, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.4, + "recallAt10": 0.6, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.851694, + "directRecallAt10": 1, + "supportingRecallAt10": 0.6 + } + }, + "oracle": { + "deliveredTokens": 1989, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 0.9, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.972313, + "directRecallAt10": 1, + "supportingRecallAt10": 0.9 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3944, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.4, + "recallAt10": 0.7, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.87938, + "directRecallAt10": 1, + "supportingRecallAt10": 0.7 + } + }, + "balancedV1": { + "deliveredTokens": 3995, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.4, + "recallAt10": 0.8, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.912415, + "directRecallAt10": 1, + "supportingRecallAt10": 0.8 + } + }, + "qarinahV2": { + "deliveredTokens": 3944, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.4, + "recallAt10": 0.7, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.87938, + "directRecallAt10": 1, + "supportingRecallAt10": 0.7 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3944, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.4, + "recallAt10": 0.7, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.87938, + "directRecallAt10": 1, + "supportingRecallAt10": 0.7 + } + }, + "oracle": { + "deliveredTokens": 2111, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7313, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.4, + "recallAt10": 0.7, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.87938, + "directRecallAt10": 1, + "supportingRecallAt10": 0.7 + } + }, + "balancedV1": { + "deliveredTokens": 7313, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.4, + "recallAt10": 0.8, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.912415, + "directRecallAt10": 1, + "supportingRecallAt10": 0.8 + } + }, + "qarinahV2": { + "deliveredTokens": 7313, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.4, + "recallAt10": 0.7, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.87938, + "directRecallAt10": 1, + "supportingRecallAt10": 0.7 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7313, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.4, + "recallAt10": 0.7, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.87938, + "directRecallAt10": 1, + "supportingRecallAt10": 0.7 + } + }, + "oracle": { + "deliveredTokens": 2111, + "metrics": { + "recallAt1": 0.1, + "recallAt5": 0.5, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.45, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-15678", + "repositorySequence": 32, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 2871, + "balancedV1": 2293, + "qarinahV2": 2871, + "qarinahV2NoGraph": 2871, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 507, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 472, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 507, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 507, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 996, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 896, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 996, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 996, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1962, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1924, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1962, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1962, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3938, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3922, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3938, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3938, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7466, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 7466, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 7466, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7466, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.506522, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 8, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-16106", + "repositorySequence": 33, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.477967, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + }, + "balancedV1": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.636364, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.712374, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.477967, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.477967, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 2942, + "balancedV1": 2403, + "qarinahV2": 2942, + "qarinahV2NoGraph": 2942, + "oracle": 2111 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 474, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 474, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2": { + "deliveredTokens": 474, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 474, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 482, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.272727, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 957, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.330138, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 945, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.330138, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 957, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.330138, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 957, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.330138, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 969, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1973, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.272727, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.408536, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 1999, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.545455, + "precisionAt5": 0.6, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.646119, + "directRecallAt10": null, + "supportingRecallAt10": 0.545455 + } + }, + "qarinahV2": { + "deliveredTokens": 1973, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.272727, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.408536, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1973, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.272727, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.408536, + "directRecallAt10": null, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 1989, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.818182, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.936379, + "directRecallAt10": null, + "supportingRecallAt10": 0.818182 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3939, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.477967, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "balancedV1": { + "deliveredTokens": 3914, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.636364, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.712374, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 3939, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.477967, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3939, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.477967, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "oracle": { + "deliveredTokens": 2264, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7704, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.477967, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "balancedV1": { + "deliveredTokens": 7704, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.272727, + "recallAt10": 0.636364, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.712374, + "directRecallAt10": null, + "supportingRecallAt10": 0.636364 + } + }, + "qarinahV2": { + "deliveredTokens": 7704, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.477967, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7704, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.363636, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.477967, + "directRecallAt10": null, + "supportingRecallAt10": 0.363636 + } + }, + "oracle": { + "deliveredTokens": 2264, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.430882, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 8, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-16281", + "repositorySequence": 34, + "positiveUnderStructuralOracle": true, + "directRecords": 2, + "supportingRecords": 10, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.083333, + "recallAt5": 0.166667, + "recallAt10": 0.416667, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.653973, + "directRecallAt10": 1, + "supportingRecallAt10": 0.416667 + }, + "balancedV1": { + "recallAt1": 0.083333, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.853491, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2": { + "recallAt1": 0.083333, + "recallAt5": 0.166667, + "recallAt10": 0.416667, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.653973, + "directRecallAt10": 1, + "supportingRecallAt10": 0.416667 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.083333, + "recallAt5": 0.166667, + "recallAt10": 0.416667, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.653973, + "directRecallAt10": 1, + "supportingRecallAt10": 0.416667 + }, + "oracle": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.833333, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.833333 + } + }, + "volumes": { + "bm25Admitted": 2434, + "balancedV1": 2320, + "qarinahV2": 2434, + "qarinahV2NoGraph": 2434, + "oracle": 2571 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 428, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.083333, + "recallAt10": 0.083333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.080832, + "directRecallAt10": 0, + "supportingRecallAt10": 0.083333 + } + }, + "balancedV1": { + "deliveredTokens": 476, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.401123, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2": { + "deliveredTokens": 428, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.083333, + "recallAt10": 0.083333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.080832, + "directRecallAt10": 0, + "supportingRecallAt10": 0.083333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 428, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.083333, + "recallAt10": 0.083333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.080832, + "directRecallAt10": 0, + "supportingRecallAt10": 0.083333 + } + }, + "oracle": { + "deliveredTokens": 478, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.529239, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.25 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.448406, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.166667 + } + }, + "balancedV1": { + "deliveredTokens": 907, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.529239, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.448406, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.448406, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.166667 + } + }, + "oracle": { + "deliveredTokens": 994, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.74608, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1935, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.166667, + "recallAt10": 0.25, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.576523, + "directRecallAt10": 1, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 1967, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.333333, + "recallAt10": 0.5, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.77789, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 1935, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.166667, + "recallAt10": 0.25, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.576523, + "directRecallAt10": 1, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1935, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.166667, + "recallAt10": 0.25, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.576523, + "directRecallAt10": 1, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 1981, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.583333, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.883983, + "directRecallAt10": 1, + "supportingRecallAt10": 0.583333 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3997, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.166667, + "recallAt10": 0.416667, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.653973, + "directRecallAt10": 1, + "supportingRecallAt10": 0.416667 + } + }, + "balancedV1": { + "deliveredTokens": 3984, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.853491, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3997, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.166667, + "recallAt10": 0.416667, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.653973, + "directRecallAt10": 1, + "supportingRecallAt10": 0.416667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3997, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.166667, + "recallAt10": 0.416667, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.653973, + "directRecallAt10": 1, + "supportingRecallAt10": 0.416667 + } + }, + "oracle": { + "deliveredTokens": 2865, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.833333, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.833333 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7876, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.166667, + "recallAt10": 0.416667, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.653973, + "directRecallAt10": 1, + "supportingRecallAt10": 0.416667 + } + }, + "balancedV1": { + "deliveredTokens": 7876, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.853491, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 7876, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.166667, + "recallAt10": 0.416667, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.653973, + "directRecallAt10": 1, + "supportingRecallAt10": 0.416667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7876, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.166667, + "recallAt10": 0.416667, + "precisionAt5": 0.4, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.653973, + "directRecallAt10": 1, + "supportingRecallAt10": 0.416667 + } + }, + "oracle": { + "deliveredTokens": 2865, + "metrics": { + "recallAt1": 0.083333, + "recallAt5": 0.416667, + "recallAt10": 0.833333, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.833333 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.226154, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "NO_CODE_ENTITY_MATCH", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-16503", + "repositorySequence": 35, + "positiveUnderStructuralOracle": true, + "directRecords": 3, + "supportingRecords": 10, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.076923, + "recallAt5": 0.230769, + "recallAt10": 0.538462, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.779183, + "directRecallAt10": 1, + "supportingRecallAt10": 0.538462 + }, + "balancedV1": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.769231, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.769231 + }, + "qarinahV2": { + "recallAt1": 0.076923, + "recallAt5": 0.230769, + "recallAt10": 0.538462, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.779183, + "directRecallAt10": 1, + "supportingRecallAt10": 0.538462 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.076923, + "recallAt5": 0.230769, + "recallAt10": 0.538462, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.779183, + "directRecallAt10": 1, + "supportingRecallAt10": 0.538462 + }, + "oracle": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.769231, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.769231 + } + }, + "volumes": { + "bm25Admitted": 2637, + "balancedV1": 2334, + "qarinahV2": 2637, + "qarinahV2NoGraph": 2637, + "oracle": 2880 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 435, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.076923, + "recallAt10": 0.076923, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.340699, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.076923 + } + }, + "balancedV1": { + "deliveredTokens": 435, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.076923, + "recallAt10": 0.076923, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.340699, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.076923 + } + }, + "qarinahV2": { + "deliveredTokens": 435, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.076923, + "recallAt10": 0.076923, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.340699, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.076923 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 435, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.076923, + "recallAt10": 0.076923, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.340699, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.076923 + } + }, + "oracle": { + "deliveredTokens": 435, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.076923, + "recallAt10": 0.076923, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.340699, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.076923 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 996, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.076923, + "recallAt10": 0.076923, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.340699, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.076923 + } + }, + "balancedV1": { + "deliveredTokens": 988, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.153846, + "recallAt10": 0.153846, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.555657, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.153846 + } + }, + "qarinahV2": { + "deliveredTokens": 996, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.076923, + "recallAt10": 0.076923, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.340699, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.076923 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 996, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.076923, + "recallAt10": 0.076923, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.340699, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.076923 + } + }, + "oracle": { + "deliveredTokens": 988, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.153846, + "recallAt10": 0.153846, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.555657, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.153846 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1884, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.230769, + "recallAt10": 0.307692, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.676342, + "directRecallAt10": 1, + "supportingRecallAt10": 0.307692 + } + }, + "balancedV1": { + "deliveredTokens": 1981, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.615385, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.932985, + "directRecallAt10": 1, + "supportingRecallAt10": 0.615385 + } + }, + "qarinahV2": { + "deliveredTokens": 1884, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.230769, + "recallAt10": 0.307692, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.676342, + "directRecallAt10": 1, + "supportingRecallAt10": 0.307692 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1884, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.230769, + "recallAt10": 0.307692, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.676342, + "directRecallAt10": 1, + "supportingRecallAt10": 0.307692 + } + }, + "oracle": { + "deliveredTokens": 1979, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.461538, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.859303, + "directRecallAt10": 1, + "supportingRecallAt10": 0.461538 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3949, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.230769, + "recallAt10": 0.538462, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.779183, + "directRecallAt10": 1, + "supportingRecallAt10": 0.538462 + } + }, + "balancedV1": { + "deliveredTokens": 3978, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.769231, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.769231 + } + }, + "qarinahV2": { + "deliveredTokens": 3949, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.230769, + "recallAt10": 0.538462, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.779183, + "directRecallAt10": 1, + "supportingRecallAt10": 0.538462 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3949, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.230769, + "recallAt10": 0.538462, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.779183, + "directRecallAt10": 1, + "supportingRecallAt10": 0.538462 + } + }, + "oracle": { + "deliveredTokens": 3300, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.769231, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.769231 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7465, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.230769, + "recallAt10": 0.538462, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.779183, + "directRecallAt10": 1, + "supportingRecallAt10": 0.538462 + } + }, + "balancedV1": { + "deliveredTokens": 7983, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.769231, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.769231 + } + }, + "qarinahV2": { + "deliveredTokens": 7894, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.230769, + "recallAt10": 0.538462, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.779183, + "directRecallAt10": 1, + "supportingRecallAt10": 0.538462 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7894, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.230769, + "recallAt10": 0.538462, + "precisionAt5": 0.6, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.779183, + "directRecallAt10": 1, + "supportingRecallAt10": 0.538462 + } + }, + "oracle": { + "deliveredTokens": 3300, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.769231, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.769231 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "DIRECTLY_SUPPORTED", + "decision": "ACCEPT_DIRECT", + "score": 0.775, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "CONSERVATIVE_DIRECT_THRESHOLD_MET" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-16792", + "repositorySequence": 36, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2739, + "balancedV1": 2629, + "qarinahV2": 2739, + "qarinahV2NoGraph": 2739, + "oracle": 242 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 463, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 444, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 463, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 463, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.63093, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 242, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 242, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1997, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1950, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1997, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1997, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 242, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3990, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3903, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3990, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3990, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 242, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7990, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 7955, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7990, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7990, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 242, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.389646, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 8, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-16988", + "repositorySequence": 37, + "positiveUnderStructuralOracle": true, + "directRecords": 2, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.177239, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.177239, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.177239, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2489, + "balancedV1": 2248, + "qarinahV2": 2489, + "qarinahV2NoGraph": 2489, + "oracle": 410 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 463, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 504, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 463, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 463, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 410, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 891, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 939, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 891, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 891, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 410, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1996, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1996, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1996, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 410, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3932, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.177239, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 3990, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3932, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.177239, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3932, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.177239, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 410, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7974, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.177239, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 7974, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7974, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.177239, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7974, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.177239, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 410, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.362069, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-17022", + "repositorySequence": 38, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 13, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.071429, + "recallAt10": 0.357143, + "precisionAt5": 0.2, + "precisionAt10": 0.5, + "reciprocalRank": 0.25, + "ndcgAt10": 0.258643, + "directRecallAt10": 0, + "supportingRecallAt10": 0.357143 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.285714, + "recallAt10": 0.642857, + "precisionAt5": 0.8, + "precisionAt10": 0.9, + "reciprocalRank": 0.5, + "ndcgAt10": 0.629885, + "directRecallAt10": 1, + "supportingRecallAt10": 0.642857 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.071429, + "recallAt10": 0.357143, + "precisionAt5": 0.2, + "precisionAt10": 0.5, + "reciprocalRank": 0.25, + "ndcgAt10": 0.258643, + "directRecallAt10": 0, + "supportingRecallAt10": 0.357143 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.071429, + "recallAt10": 0.357143, + "precisionAt5": 0.2, + "precisionAt10": 0.5, + "reciprocalRank": 0.25, + "ndcgAt10": 0.258643, + "directRecallAt10": 0, + "supportingRecallAt10": 0.357143 + }, + "oracle": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.714286, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.714286 + } + }, + "volumes": { + "bm25Admitted": 3252, + "balancedV1": 2442, + "qarinahV2": 3252, + "qarinahV2NoGraph": 3252, + "oracle": 2472 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 472, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.09642, + "directRecallAt10": 0, + "supportingRecallAt10": 0.071429 + } + }, + "balancedV1": { + "deliveredTokens": 488, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.172831, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2": { + "deliveredTokens": 472, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.09642, + "directRecallAt10": 0, + "supportingRecallAt10": 0.071429 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 472, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.071429, + "recallAt10": 0.071429, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.09642, + "directRecallAt10": 0, + "supportingRecallAt10": 0.071429 + } + }, + "oracle": { + "deliveredTokens": 409, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.214286, + "recallAt10": 0.214286, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.631297, + "directRecallAt10": 1, + "supportingRecallAt10": 0.214286 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 894, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 915, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.172831, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2": { + "deliveredTokens": 894, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 894, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 994, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.697114, + "directRecallAt10": 1, + "supportingRecallAt10": 0.285714 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.071429, + "recallAt10": 0.214286, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.25, + "ndcgAt10": 0.164968, + "directRecallAt10": 0, + "supportingRecallAt10": 0.214286 + } + }, + "balancedV1": { + "deliveredTokens": 1998, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.285714, + "recallAt10": 0.5, + "precisionAt5": 0.8, + "precisionAt10": 0.7, + "reciprocalRank": 0.5, + "ndcgAt10": 0.451354, + "directRecallAt10": 0, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 1997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.071429, + "recallAt10": 0.214286, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.25, + "ndcgAt10": 0.164968, + "directRecallAt10": 0, + "supportingRecallAt10": 0.214286 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.071429, + "recallAt10": 0.214286, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.25, + "ndcgAt10": 0.164968, + "directRecallAt10": 0, + "supportingRecallAt10": 0.214286 + } + }, + "oracle": { + "deliveredTokens": 1882, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.5, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.861611, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.071429, + "recallAt10": 0.357143, + "precisionAt5": 0.2, + "precisionAt10": 0.5, + "reciprocalRank": 0.25, + "ndcgAt10": 0.258643, + "directRecallAt10": 0, + "supportingRecallAt10": 0.357143 + } + }, + "balancedV1": { + "deliveredTokens": 3898, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.285714, + "recallAt10": 0.642857, + "precisionAt5": 0.8, + "precisionAt10": 0.9, + "reciprocalRank": 0.5, + "ndcgAt10": 0.629885, + "directRecallAt10": 1, + "supportingRecallAt10": 0.642857 + } + }, + "qarinahV2": { + "deliveredTokens": 3970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.071429, + "recallAt10": 0.357143, + "precisionAt5": 0.2, + "precisionAt10": 0.5, + "reciprocalRank": 0.25, + "ndcgAt10": 0.258643, + "directRecallAt10": 0, + "supportingRecallAt10": 0.357143 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.071429, + "recallAt10": 0.357143, + "precisionAt5": 0.2, + "precisionAt10": 0.5, + "reciprocalRank": 0.25, + "ndcgAt10": 0.258643, + "directRecallAt10": 0, + "supportingRecallAt10": 0.357143 + } + }, + "oracle": { + "deliveredTokens": 3414, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.714286, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.714286 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7968, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.071429, + "recallAt10": 0.357143, + "precisionAt5": 0.2, + "precisionAt10": 0.5, + "reciprocalRank": 0.25, + "ndcgAt10": 0.258643, + "directRecallAt10": 0, + "supportingRecallAt10": 0.357143 + } + }, + "balancedV1": { + "deliveredTokens": 7945, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.285714, + "recallAt10": 0.642857, + "precisionAt5": 0.8, + "precisionAt10": 0.9, + "reciprocalRank": 0.5, + "ndcgAt10": 0.629885, + "directRecallAt10": 1, + "supportingRecallAt10": 0.642857 + } + }, + "qarinahV2": { + "deliveredTokens": 7968, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.071429, + "recallAt10": 0.357143, + "precisionAt5": 0.2, + "precisionAt10": 0.5, + "reciprocalRank": 0.25, + "ndcgAt10": 0.258643, + "directRecallAt10": 0, + "supportingRecallAt10": 0.357143 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7968, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.071429, + "recallAt10": 0.357143, + "precisionAt5": 0.2, + "precisionAt10": 0.5, + "reciprocalRank": 0.25, + "ndcgAt10": 0.258643, + "directRecallAt10": 0, + "supportingRecallAt10": 0.357143 + } + }, + "oracle": { + "deliveredTokens": 3414, + "metrics": { + "recallAt1": 0.071429, + "recallAt5": 0.357143, + "recallAt10": 0.714286, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.714286 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.501726, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-17139", + "repositorySequence": 39, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3388, + "balancedV1": 2663, + "qarinahV2": 3388, + "qarinahV2NoGraph": 3388, + "oracle": 339 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 461, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 481, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 461, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 461, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 339, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 892, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 996, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 892, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 892, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 339, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1982, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1998, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1982, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1982, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 339, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3970, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3904, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3970, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3970, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 339, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7978, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 7901, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7978, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7978, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 339, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.447143, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 7, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-17630", + "repositorySequence": 40, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 4, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.8, + "precisionAt5": 0.2, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.532678, + "directRecallAt10": 1, + "supportingRecallAt10": 0.8 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.188074, + "directRecallAt10": 0, + "supportingRecallAt10": 0.4 + }, + "qarinahV2": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.8, + "precisionAt5": 0.2, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.532678, + "directRecallAt10": 1, + "supportingRecallAt10": 0.8 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.8, + "precisionAt5": 0.2, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.532678, + "directRecallAt10": 1, + "supportingRecallAt10": 0.8 + }, + "oracle": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3738, + "balancedV1": 2672, + "qarinahV2": 3738, + "qarinahV2NoGraph": 3738, + "oracle": 1495 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 508, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.202083, + "directRecallAt10": 0, + "supportingRecallAt10": 0.2 + } + }, + "balancedV1": { + "deliveredTokens": 472, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.1275, + "directRecallAt10": 0, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2": { + "deliveredTokens": 508, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.202083, + "directRecallAt10": 0, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 508, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.202083, + "directRecallAt10": 0, + "supportingRecallAt10": 0.2 + } + }, + "oracle": { + "deliveredTokens": 346, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.606249, + "directRecallAt10": 1, + "supportingRecallAt10": 0.2 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 942, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.303125, + "directRecallAt10": 0, + "supportingRecallAt10": 0.4 + } + }, + "balancedV1": { + "deliveredTokens": 945, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 942, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.303125, + "directRecallAt10": 0, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 942, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.303125, + "directRecallAt10": 0, + "supportingRecallAt10": 0.4 + } + }, + "oracle": { + "deliveredTokens": 893, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.6, + "recallAt10": 0.6, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.834791, + "directRecallAt10": 1, + "supportingRecallAt10": 0.6 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1922, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.28026, + "directRecallAt10": 0, + "supportingRecallAt10": 0.4 + } + }, + "balancedV1": { + "deliveredTokens": 1927, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.188074, + "directRecallAt10": 0, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2": { + "deliveredTokens": 1922, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.28026, + "directRecallAt10": 0, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1922, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.28026, + "directRecallAt10": 0, + "supportingRecallAt10": 0.4 + } + }, + "oracle": { + "deliveredTokens": 1495, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3975, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.8, + "precisionAt5": 0.2, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.532678, + "directRecallAt10": 1, + "supportingRecallAt10": 0.8 + } + }, + "balancedV1": { + "deliveredTokens": 3912, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.188074, + "directRecallAt10": 0, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2": { + "deliveredTokens": 3975, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.8, + "precisionAt5": 0.2, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.532678, + "directRecallAt10": 1, + "supportingRecallAt10": 0.8 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3975, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.8, + "precisionAt5": 0.2, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.532678, + "directRecallAt10": 1, + "supportingRecallAt10": 0.8 + } + }, + "oracle": { + "deliveredTokens": 1495, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7992, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.8, + "precisionAt5": 0.2, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.532678, + "directRecallAt10": 1, + "supportingRecallAt10": 0.8 + } + }, + "balancedV1": { + "deliveredTokens": 7998, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.4, + "recallAt10": 0.4, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.188074, + "directRecallAt10": 0, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2": { + "deliveredTokens": 7992, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.8, + "precisionAt5": 0.2, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.532678, + "directRecallAt10": 1, + "supportingRecallAt10": 0.8 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7992, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.8, + "precisionAt5": 0.2, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.532678, + "directRecallAt10": 1, + "supportingRecallAt10": 0.8 + } + }, + "oracle": { + "deliveredTokens": 1495, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.417703, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-17655", + "repositorySequence": 41, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4801, + "balancedV1": 3450, + "qarinahV2": 4801, + "qarinahV2NoGraph": 4801, + "oracle": 238 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 489, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 451, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 489, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 489, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 238, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 920, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 920, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 920, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 920, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 238, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1962, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1924, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1962, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1962, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.5, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 238, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3904, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3890, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3904, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3904, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 238, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7992, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 7946, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7992, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7992, + "metrics": { + "recallAt1": 0, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.430677, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 238, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.416071, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-18057", + "repositorySequence": 42, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 6, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.166667, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.100867, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.117063, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.166667, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.100867, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.166667, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.100867, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + }, + "oracle": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4545, + "balancedV1": 3507, + "qarinahV2": 4545, + "qarinahV2NoGraph": 4545, + "oracle": 1163 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 474, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.190921, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "balancedV1": { + "deliveredTokens": 487, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 474, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.190921, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 474, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.190921, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "oracle": { + "deliveredTokens": 455, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.493523, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 976, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 899, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 976, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 976, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 971, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 0.833333, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.892211, + "directRecallAt10": null, + "supportingRecallAt10": 0.833333 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1947, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.130324, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "balancedV1": { + "deliveredTokens": 1960, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1947, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.130324, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1947, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.130324, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "oracle": { + "deliveredTokens": 1163, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3987, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.166667, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.100867, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "balancedV1": { + "deliveredTokens": 3974, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.117063, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2": { + "deliveredTokens": 3987, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.166667, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.100867, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3987, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.166667, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.100867, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "oracle": { + "deliveredTokens": 1163, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7908, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.166667, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.100867, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "balancedV1": { + "deliveredTokens": 7902, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.117063, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2": { + "deliveredTokens": 7908, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.166667, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.100867, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7908, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.166667, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.100867, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "oracle": { + "deliveredTokens": 1163, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.580169, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 8, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-18087", + "repositorySequence": 43, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 7, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.428571, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.322065, + "directRecallAt10": null, + "supportingRecallAt10": 0.428571 + }, + "balancedV1": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.393259, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.428571, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.322065, + "directRecallAt10": null, + "supportingRecallAt10": 0.428571 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.428571, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.322065, + "directRecallAt10": null, + "supportingRecallAt10": 0.428571 + }, + "oracle": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3473, + "balancedV1": 3363, + "qarinahV2": 3473, + "qarinahV2NoGraph": 3473, + "oracle": 1654 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 506, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.274876, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 499, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 455, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.448304, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 993, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.137438, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "balancedV1": { + "deliveredTokens": 997, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.393259, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + }, + "qarinahV2": { + "deliveredTokens": 993, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.137438, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 993, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.137438, + "directRecallAt10": null, + "supportingRecallAt10": 0.142857 + } + }, + "oracle": { + "deliveredTokens": 946, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.428571, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.585742, + "directRecallAt10": null, + "supportingRecallAt10": 0.428571 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1914, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.285714, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.235351, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + }, + "balancedV1": { + "deliveredTokens": 1896, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.393259, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + }, + "qarinahV2": { + "deliveredTokens": 1914, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.285714, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.235351, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1914, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.285714, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.235351, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + }, + "oracle": { + "deliveredTokens": 1654, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3961, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.428571, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.322065, + "directRecallAt10": null, + "supportingRecallAt10": 0.428571 + } + }, + "balancedV1": { + "deliveredTokens": 3975, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.393259, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + }, + "qarinahV2": { + "deliveredTokens": 3961, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.428571, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.322065, + "directRecallAt10": null, + "supportingRecallAt10": 0.428571 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3961, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.428571, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.322065, + "directRecallAt10": null, + "supportingRecallAt10": 0.428571 + } + }, + "oracle": { + "deliveredTokens": 1654, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7969, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.428571, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.322065, + "directRecallAt10": null, + "supportingRecallAt10": 0.428571 + } + }, + "balancedV1": { + "deliveredTokens": 7943, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.393259, + "directRecallAt10": null, + "supportingRecallAt10": 0.285714 + } + }, + "qarinahV2": { + "deliveredTokens": 7969, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.428571, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.322065, + "directRecallAt10": null, + "supportingRecallAt10": 0.428571 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7969, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.428571, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.322065, + "directRecallAt10": null, + "supportingRecallAt10": 0.428571 + } + }, + "oracle": { + "deliveredTokens": 1654, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.405, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "SINGLE_OR_NO_SUPPORT", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-18189", + "repositorySequence": 44, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 2816, + "balancedV1": 3337, + "qarinahV2": 2816, + "qarinahV2NoGraph": 2816, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 505, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 512, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 505, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 505, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 969, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 950, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 969, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 969, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1935, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1897, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1935, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1935, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3957, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3916, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3957, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3957, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7993, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 7993, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 7993, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7993, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.585065, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 6, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-18199", + "repositorySequence": 45, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 3003, + "balancedV1": 2897, + "qarinahV2": 3003, + "qarinahV2NoGraph": 3003, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 486, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 421, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 486, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 486, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 948, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 984, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 948, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 948, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1961, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1977, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1961, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1961, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3907, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3935, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3907, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3907, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7963, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 7933, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 7963, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7963, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.515789, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-18532", + "repositorySequence": 46, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 8, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.159589, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.159589, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.159589, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + }, + "oracle": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3758, + "balancedV1": 3578, + "qarinahV2": 3758, + "qarinahV2NoGraph": 3758, + "oracle": 1786 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.252943, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + }, + "balancedV1": { + "deliveredTokens": 444, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.252943, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.252943, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + }, + "oracle": { + "deliveredTokens": 455, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.412532, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 912, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 912, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 912, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 946, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.375, + "recallAt10": 0.375, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.539003, + "directRecallAt10": null, + "supportingRecallAt10": 0.375 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.125, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.243903, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 1917, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.125, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.243903, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1984, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.125, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.243903, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 1786, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3973, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.159589, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + }, + "balancedV1": { + "deliveredTokens": 3995, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3973, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.159589, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3973, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.159589, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + }, + "oracle": { + "deliveredTokens": 1786, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.159589, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + }, + "balancedV1": { + "deliveredTokens": 7893, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.159589, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.125, + "recallAt10": 0.125, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.159589, + "directRecallAt10": null, + "supportingRecallAt10": 0.125 + } + }, + "oracle": { + "deliveredTokens": 1786, + "metrics": { + "recallAt1": 0.125, + "recallAt5": 0.625, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.598, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-18621", + "repositorySequence": 47, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 6, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.166667, + "recallAt5": 0.5, + "recallAt10": 0.666667, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.671834, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "balancedV1": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.775148, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2": { + "recallAt1": 0.166667, + "recallAt5": 0.5, + "recallAt10": 0.666667, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.671834, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.166667, + "recallAt5": 0.5, + "recallAt10": 0.666667, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.671834, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "oracle": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4911, + "balancedV1": 3566, + "qarinahV2": 4911, + "qarinahV2NoGraph": 4911, + "oracle": 2415 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 504, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.302602, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "balancedV1": { + "deliveredTokens": 444, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.302602, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2": { + "deliveredTokens": 504, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.302602, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 504, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.302602, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "oracle": { + "deliveredTokens": 326, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.302602, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 920, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.302602, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "balancedV1": { + "deliveredTokens": 920, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.302602, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2": { + "deliveredTokens": 920, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.302602, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 920, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.302602, + "directRecallAt10": null, + "supportingRecallAt10": 0.166667 + } + }, + "oracle": { + "deliveredTokens": 912, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.644824, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.453904, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 1954, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.775148, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.453904, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.453904, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 1957, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.775148, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3965, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.5, + "recallAt10": 0.666667, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.671834, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 3998, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.775148, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3965, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.5, + "recallAt10": 0.666667, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.671834, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3965, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.5, + "recallAt10": 0.666667, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.671834, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 2415, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7989, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.5, + "recallAt10": 0.666667, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.671834, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 7968, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.775148, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 7989, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.5, + "recallAt10": 0.666667, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.671834, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7989, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.5, + "recallAt10": 0.666667, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.671834, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 2415, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.608553, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-18698", + "repositorySequence": 48, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.193426, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.193426, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.193426, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2827, + "balancedV1": 2501, + "qarinahV2": 2827, + "qarinahV2NoGraph": 2827, + "oracle": 1092 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 486, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 469, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 486, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 486, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 0, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 895, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 959, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 895, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 895, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 572, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1948, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 1092, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3993, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.193426, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 3985, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3993, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.193426, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3993, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.193426, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 1092, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7934, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.193426, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 7991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7934, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.193426, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7934, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.193426, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 1092, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.392105, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "NO_CODE_ENTITY_MATCH", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-18835", + "repositorySequence": 49, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.177239, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3942, + "balancedV1": 3861, + "qarinahV2": 3942, + "qarinahV2NoGraph": 3942, + "oracle": 790 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 503, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 457, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 503, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 503, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 242, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 900, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 896, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 900, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 900, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 790, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1937, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 1993, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1937, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1937, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 790, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3942, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 3975, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.177239, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 3942, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3942, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 790, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7956, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 7991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.177239, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 7956, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7956, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.386853, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 790, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.542105, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-19007", + "repositorySequence": 50, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 6, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.285714, + "recallAt10": 0.571429, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.25, + "ndcgAt10": 0.412856, + "directRecallAt10": 1, + "supportingRecallAt10": 0.571429 + }, + "balancedV1": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.428571, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.329232, + "directRecallAt10": 0, + "supportingRecallAt10": 0.428571 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.285714, + "recallAt10": 0.571429, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.25, + "ndcgAt10": 0.412856, + "directRecallAt10": 1, + "supportingRecallAt10": 0.571429 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.285714, + "recallAt10": 0.571429, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.25, + "ndcgAt10": 0.412856, + "directRecallAt10": 1, + "supportingRecallAt10": 0.571429 + }, + "oracle": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3691, + "balancedV1": 2964, + "qarinahV2": 3691, + "qarinahV2NoGraph": 3691, + "oracle": 2999 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 431, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 472, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.177368, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2": { + "deliveredTokens": 431, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 431, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 326, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.177368, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 894, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.076388, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + } + }, + "balancedV1": { + "deliveredTokens": 943, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.177368, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2": { + "deliveredTokens": 894, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.076388, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 894, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.142857, + "recallAt10": 0.142857, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.076388, + "directRecallAt10": 0, + "supportingRecallAt10": 0.142857 + } + }, + "oracle": { + "deliveredTokens": 910, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.285714, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.64401, + "directRecallAt10": 1, + "supportingRecallAt10": 0.285714 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1993, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.285714, + "recallAt10": 0.428571, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.25, + "ndcgAt10": 0.36096, + "directRecallAt10": 1, + "supportingRecallAt10": 0.428571 + } + }, + "balancedV1": { + "deliveredTokens": 1931, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.428571, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.329232, + "directRecallAt10": 0, + "supportingRecallAt10": 0.428571 + } + }, + "qarinahV2": { + "deliveredTokens": 1993, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.285714, + "recallAt10": 0.428571, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.25, + "ndcgAt10": 0.36096, + "directRecallAt10": 1, + "supportingRecallAt10": 0.428571 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1993, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.285714, + "recallAt10": 0.428571, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.25, + "ndcgAt10": 0.36096, + "directRecallAt10": 1, + "supportingRecallAt10": 0.428571 + } + }, + "oracle": { + "deliveredTokens": 1830, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.428571, + "recallAt10": 0.428571, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.732694, + "directRecallAt10": 1, + "supportingRecallAt10": 0.428571 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3982, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.285714, + "recallAt10": 0.571429, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.25, + "ndcgAt10": 0.412856, + "directRecallAt10": 1, + "supportingRecallAt10": 0.571429 + } + }, + "balancedV1": { + "deliveredTokens": 3940, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.428571, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.329232, + "directRecallAt10": 0, + "supportingRecallAt10": 0.428571 + } + }, + "qarinahV2": { + "deliveredTokens": 3982, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.285714, + "recallAt10": 0.571429, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.25, + "ndcgAt10": 0.412856, + "directRecallAt10": 1, + "supportingRecallAt10": 0.571429 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3982, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.285714, + "recallAt10": 0.571429, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.25, + "ndcgAt10": 0.412856, + "directRecallAt10": 1, + "supportingRecallAt10": 0.571429 + } + }, + "oracle": { + "deliveredTokens": 2999, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7896, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.285714, + "recallAt10": 0.571429, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.25, + "ndcgAt10": 0.412856, + "directRecallAt10": 1, + "supportingRecallAt10": 0.571429 + } + }, + "balancedV1": { + "deliveredTokens": 7950, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.285714, + "recallAt10": 0.428571, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.329232, + "directRecallAt10": 0, + "supportingRecallAt10": 0.428571 + } + }, + "qarinahV2": { + "deliveredTokens": 7896, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.285714, + "recallAt10": 0.571429, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.25, + "ndcgAt10": 0.412856, + "directRecallAt10": 1, + "supportingRecallAt10": 0.571429 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7896, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.285714, + "recallAt10": 0.571429, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.25, + "ndcgAt10": 0.412856, + "directRecallAt10": 1, + "supportingRecallAt10": 0.571429 + } + }, + "oracle": { + "deliveredTokens": 2999, + "metrics": { + "recallAt1": 0.142857, + "recallAt5": 0.714286, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "DIRECTLY_SUPPORTED", + "decision": "ACCEPT_DIRECT", + "score": 0.708333, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "CONSERVATIVE_DIRECT_THRESHOLD_MET" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-19254", + "repositorySequence": 51, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.38268, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.38268, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.38268, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3400, + "balancedV1": 3278, + "qarinahV2": 3400, + "qarinahV2NoGraph": 3400, + "oracle": 1261 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 435, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 435, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 435, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 435, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 169, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.469279, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 926, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 988, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 926, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 926, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 741, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1948, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.234639, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1948, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.234639, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1948, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.234639, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 1261, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3953, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.38268, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 3988, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3953, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.38268, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3953, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.38268, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 1261, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7985, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.38268, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 7917, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7985, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.38268, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7985, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.38268, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 1261, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.34078, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-19487", + "repositorySequence": 52, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.202107, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.202107, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.202107, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3678, + "balancedV1": 3255, + "qarinahV2": 3678, + "qarinahV2NoGraph": 3678, + "oracle": 542 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 505, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 435, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 505, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 505, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 382, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.202107, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 890, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.202107, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.202107, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 542, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1915, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.202107, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 1942, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1915, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.202107, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1915, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.202107, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 542, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3933, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.202107, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 3985, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3933, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.202107, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3933, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.202107, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 542, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7954, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.202107, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 7992, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7954, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.202107, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7954, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.25, + "ndcgAt10": 0.202107, + "directRecallAt10": null, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 542, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.227143, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "NO_CODE_ENTITY_MATCH", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-20049", + "repositorySequence": 53, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 4807, + "balancedV1": 3901, + "qarinahV2": 4807, + "qarinahV2NoGraph": 4807, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 468, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 472, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 468, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 468, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 964, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 920, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 964, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 964, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1938, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1945, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1938, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1938, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3994, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3901, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3994, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3994, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7943, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 7999, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 7943, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7943, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.413736, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-20154", + "repositorySequence": 54, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.830485, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.367173, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.830485, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.830485, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3213, + "balancedV1": 2829, + "qarinahV2": 3213, + "qarinahV2NoGraph": 3213, + "oracle": 1037 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 417, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.726229, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 435, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 417, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.726229, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 417, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.726229, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 489, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 937, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.726229, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 988, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 937, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.726229, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 937, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.726229, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 795, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.878962, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1920, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.830485, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 1888, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.280944, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 1920, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.830485, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1920, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.830485, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 1037, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3991, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.830485, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 3888, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.367173, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3991, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.830485, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3991, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.830485, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 1037, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7955, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.830485, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 7921, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.367173, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 7955, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.830485, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7955, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.830485, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 1037, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.64, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-20212", + "repositorySequence": 55, + "positiveUnderStructuralOracle": true, + "directRecords": 3, + "supportingRecords": 8, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.363636, + "precisionAt5": 0.2, + "precisionAt10": 0.4, + "reciprocalRank": 0.25, + "ndcgAt10": 0.15542, + "directRecallAt10": 0, + "supportingRecallAt10": 0.363636 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.363636, + "precisionAt5": 0.2, + "precisionAt10": 0.4, + "reciprocalRank": 0.25, + "ndcgAt10": 0.15542, + "directRecallAt10": 0, + "supportingRecallAt10": 0.363636 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.363636, + "precisionAt5": 0.2, + "precisionAt10": 0.4, + "reciprocalRank": 0.25, + "ndcgAt10": 0.15542, + "directRecallAt10": 0, + "supportingRecallAt10": 0.363636 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 3364, + "balancedV1": 3643, + "qarinahV2": 3364, + "qarinahV2NoGraph": 3364, + "oracle": 2097 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 445, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.071652, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 460, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 445, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.071652, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 445, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.071652, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 407, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.555657, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 993, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.056783, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 899, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 993, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.056783, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 993, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.056783, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 928, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.81885, + "directRecallAt10": 1, + "supportingRecallAt10": 0.454545 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1974, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.272727, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.25, + "ndcgAt10": 0.127219, + "directRecallAt10": 0, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 1996, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1974, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.272727, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.25, + "ndcgAt10": 0.127219, + "directRecallAt10": 0, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1974, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.272727, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.25, + "ndcgAt10": 0.127219, + "directRecallAt10": 0, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 1905, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.818182, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.967172, + "directRecallAt10": 1, + "supportingRecallAt10": 0.818182 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3995, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.363636, + "precisionAt5": 0.2, + "precisionAt10": 0.4, + "reciprocalRank": 0.25, + "ndcgAt10": 0.15542, + "directRecallAt10": 0, + "supportingRecallAt10": 0.363636 + } + }, + "balancedV1": { + "deliveredTokens": 3983, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3995, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.363636, + "precisionAt5": 0.2, + "precisionAt10": 0.4, + "reciprocalRank": 0.25, + "ndcgAt10": 0.15542, + "directRecallAt10": 0, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3995, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.363636, + "precisionAt5": 0.2, + "precisionAt10": 0.4, + "reciprocalRank": 0.25, + "ndcgAt10": 0.15542, + "directRecallAt10": 0, + "supportingRecallAt10": 0.363636 + } + }, + "oracle": { + "deliveredTokens": 2240, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7913, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.363636, + "precisionAt5": 0.2, + "precisionAt10": 0.4, + "reciprocalRank": 0.25, + "ndcgAt10": 0.15542, + "directRecallAt10": 0, + "supportingRecallAt10": 0.363636 + } + }, + "balancedV1": { + "deliveredTokens": 7880, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7913, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.363636, + "precisionAt5": 0.2, + "precisionAt10": 0.4, + "reciprocalRank": 0.25, + "ndcgAt10": 0.15542, + "directRecallAt10": 0, + "supportingRecallAt10": 0.363636 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7913, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.363636, + "precisionAt5": 0.2, + "precisionAt10": 0.4, + "reciprocalRank": 0.25, + "ndcgAt10": 0.15542, + "directRecallAt10": 0, + "supportingRecallAt10": 0.363636 + } + }, + "oracle": { + "deliveredTokens": 2240, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.365714, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "NO_CODE_ENTITY_MATCH", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-20322", + "repositorySequence": 56, + "positiveUnderStructuralOracle": true, + "directRecords": 2, + "supportingRecords": 9, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.272727, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.25, + "ndcgAt10": 0.134916, + "directRecallAt10": 0, + "supportingRecallAt10": 0.272727 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.11362, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.272727, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.25, + "ndcgAt10": 0.134916, + "directRecallAt10": 0, + "supportingRecallAt10": 0.272727 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.272727, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.25, + "ndcgAt10": 0.134916, + "directRecallAt10": 0, + "supportingRecallAt10": 0.272727 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 3818, + "balancedV1": 3472, + "qarinahV2": 3818, + "qarinahV2NoGraph": 3818, + "oracle": 2318 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 455, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 421, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.080832, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2": { + "deliveredTokens": 455, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 455, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 455, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.465181, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 895, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.064058, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 974, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.064058, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2": { + "deliveredTokens": 895, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.064058, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 895, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.064058, + "directRecallAt10": 0, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 991, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.363636, + "recallAt10": 0.363636, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.74608, + "directRecallAt10": 1, + "supportingRecallAt10": 0.363636 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1939, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.25, + "ndcgAt10": 0.104739, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "balancedV1": { + "deliveredTokens": 1959, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.11362, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 1939, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.25, + "ndcgAt10": 0.104739, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1939, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.25, + "ndcgAt10": 0.104739, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "oracle": { + "deliveredTokens": 1987, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.727273, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.924399, + "directRecallAt10": 1, + "supportingRecallAt10": 0.727273 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3959, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.272727, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.25, + "ndcgAt10": 0.134916, + "directRecallAt10": 0, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 3963, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.11362, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 3959, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.272727, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.25, + "ndcgAt10": 0.134916, + "directRecallAt10": 0, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3959, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.272727, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.25, + "ndcgAt10": 0.134916, + "directRecallAt10": 0, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 2461, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7977, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.272727, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.25, + "ndcgAt10": 0.134916, + "directRecallAt10": 0, + "supportingRecallAt10": 0.272727 + } + }, + "balancedV1": { + "deliveredTokens": 7991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.11362, + "directRecallAt10": 0, + "supportingRecallAt10": 0.181818 + } + }, + "qarinahV2": { + "deliveredTokens": 7977, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.272727, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.25, + "ndcgAt10": 0.134916, + "directRecallAt10": 0, + "supportingRecallAt10": 0.272727 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7977, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.090909, + "recallAt10": 0.272727, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.25, + "ndcgAt10": 0.134916, + "directRecallAt10": 0, + "supportingRecallAt10": 0.272727 + } + }, + "oracle": { + "deliveredTokens": 2461, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.570158, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-20442", + "repositorySequence": 57, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 2622, + "balancedV1": 2846, + "qarinahV2": 2622, + "qarinahV2NoGraph": 2622, + "oracle": 700 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 423, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 505, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 423, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 423, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 0, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 995, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 995, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 995, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 700, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1889, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1926, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1889, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1889, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 700, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3995, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 3973, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3995, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3995, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 700, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7940, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 7996, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7940, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7940, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 700, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.275, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "NO_CODE_ENTITY_MATCH", + "SINGLE_OR_NO_SUPPORT", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-20590", + "repositorySequence": 58, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.090909, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.063621, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + }, + "qarinahV2": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + }, + "oracle": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + }, + "volumes": { + "bm25Admitted": 4638, + "balancedV1": 4242, + "qarinahV2": 4638, + "qarinahV2NoGraph": 4638, + "oracle": 2174 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 510, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 455, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.181818, + "recallAt10": 0.181818, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.358954, + "directRecallAt10": null, + "supportingRecallAt10": 0.181818 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 976, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 920, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 976, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 976, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 979, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.454545, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.648932, + "directRecallAt10": null, + "supportingRecallAt10": 0.454545 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1896, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 1991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1896, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1896, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 1982, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.818182, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.936379, + "directRecallAt10": null, + "supportingRecallAt10": 0.818182 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3938, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 3998, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3938, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3938, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 2317, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7974, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "balancedV1": { + "deliveredTokens": 7946, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.090909, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.1, + "ndcgAt10": 0.063621, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2": { + "deliveredTokens": 7974, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7974, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.090909, + "recallAt10": 0.090909, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.090909 + } + }, + "oracle": { + "deliveredTokens": 2317, + "metrics": { + "recallAt1": 0.090909, + "recallAt5": 0.454545, + "recallAt10": 0.909091, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.909091 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.468182, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-20639", + "repositorySequence": 59, + "positiveUnderStructuralOracle": true, + "directRecords": 4, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.778126, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + }, + "balancedV1": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.778126, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.778126, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.778126, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + }, + "oracle": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "volumes": { + "bm25Admitted": 2322, + "balancedV1": 2322, + "qarinahV2": 2322, + "qarinahV2NoGraph": 2322, + "oracle": 2794 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 427, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.133333, + "recallAt10": 0.133333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.168715, + "directRecallAt10": 0, + "supportingRecallAt10": 0.133333 + } + }, + "balancedV1": { + "deliveredTokens": 504, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.323886, + "directRecallAt10": 0.25, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2": { + "deliveredTokens": 427, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.133333, + "recallAt10": 0.133333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.168715, + "directRecallAt10": 0, + "supportingRecallAt10": 0.133333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 427, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.133333, + "recallAt10": 0.133333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.168715, + "directRecallAt10": 0, + "supportingRecallAt10": 0.133333 + } + }, + "oracle": { + "deliveredTokens": 504, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.266667, + "recallAt10": 0.266667, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.602421, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.266667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.4, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.511, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.4 + } + }, + "balancedV1": { + "deliveredTokens": 993, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.4, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.511, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.4, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.511, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 981, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.4, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.511, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.4 + } + }, + "oracle": { + "deliveredTokens": 990, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.745887, + "directRecallAt10": 0.75, + "supportingRecallAt10": 0.333333 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1969, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.533333, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.717082, + "directRecallAt10": 1, + "supportingRecallAt10": 0.533333 + } + }, + "balancedV1": { + "deliveredTokens": 1981, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.533333, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.717082, + "directRecallAt10": 1, + "supportingRecallAt10": 0.533333 + } + }, + "qarinahV2": { + "deliveredTokens": 1969, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.533333, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.717082, + "directRecallAt10": 1, + "supportingRecallAt10": 0.533333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1969, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.533333, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.717082, + "directRecallAt10": 1, + "supportingRecallAt10": 0.533333 + } + }, + "oracle": { + "deliveredTokens": 1967, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.4, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.87184, + "directRecallAt10": 1, + "supportingRecallAt10": 0.4 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3987, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.778126, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 3917, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.778126, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 3987, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.778126, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3987, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.778126, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 3719, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7914, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.778126, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "balancedV1": { + "deliveredTokens": 7965, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.778126, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2": { + "deliveredTokens": 7914, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.778126, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7914, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 0.778126, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + }, + "oracle": { + "deliveredTokens": 3719, + "metrics": { + "recallAt1": 0.066667, + "recallAt5": 0.333333, + "recallAt10": 0.666667, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.666667 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.367097, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-21055", + "repositorySequence": 60, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 2902, + "balancedV1": 2734, + "qarinahV2": 2902, + "qarinahV2NoGraph": 2902, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 431, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 407, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 431, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 431, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 985, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 960, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 985, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 985, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1953, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1994, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1953, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1953, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3966, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3890, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3966, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3966, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 7966, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.33617, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "NO_CODE_ENTITY_MATCH", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-21171", + "repositorySequence": 61, + "positiveUnderStructuralOracle": true, + "directRecords": 6, + "supportingRecords": 10, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.0625, + "recallAt5": 0.1875, + "recallAt10": 0.1875, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.280728, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.1875 + }, + "balancedV1": { + "recallAt1": 0.0625, + "recallAt5": 0.25, + "recallAt10": 0.5, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.788359, + "directRecallAt10": 0.833333, + "supportingRecallAt10": 0.5 + }, + "qarinahV2": { + "recallAt1": 0.0625, + "recallAt5": 0.1875, + "recallAt10": 0.1875, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.280728, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.1875 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.0625, + "recallAt5": 0.1875, + "recallAt10": 0.1875, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.280728, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.1875 + }, + "oracle": { + "recallAt1": 0.0625, + "recallAt5": 0.3125, + "recallAt10": 0.625, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.625 + } + }, + "volumes": { + "bm25Admitted": 5284, + "balancedV1": 3143, + "qarinahV2": 5284, + "qarinahV2NoGraph": 5284, + "oracle": 2446 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 465, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.0625, + "recallAt10": 0.0625, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.268989, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.0625 + } + }, + "balancedV1": { + "deliveredTokens": 428, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.0625, + "recallAt10": 0.0625, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.268989, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.0625 + } + }, + "qarinahV2": { + "deliveredTokens": 465, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.0625, + "recallAt10": 0.0625, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.268989, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.0625 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 465, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.0625, + "recallAt10": 0.0625, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.268989, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.0625 + } + }, + "oracle": { + "deliveredTokens": 505, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.1875, + "recallAt10": 0.1875, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.483533, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.1875 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 992, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.1875, + "recallAt10": 0.1875, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.39387, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.1875 + } + }, + "balancedV1": { + "deliveredTokens": 996, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.623389, + "directRecallAt10": 0.666667, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2": { + "deliveredTokens": 992, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.1875, + "recallAt10": 0.1875, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.39387, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.1875 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 992, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.1875, + "recallAt10": 0.1875, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.39387, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.1875 + } + }, + "oracle": { + "deliveredTokens": 945, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.3125, + "recallAt10": 0.3125, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.793102, + "directRecallAt10": 0.833333, + "supportingRecallAt10": 0.3125 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1952, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.1875, + "recallAt10": 0.1875, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.280728, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.1875 + } + }, + "balancedV1": { + "deliveredTokens": 1989, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.25, + "recallAt10": 0.375, + "precisionAt5": 0.8, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.740184, + "directRecallAt10": 0.833333, + "supportingRecallAt10": 0.375 + } + }, + "qarinahV2": { + "deliveredTokens": 1952, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.1875, + "recallAt10": 0.1875, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.280728, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.1875 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1952, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.1875, + "recallAt10": 0.1875, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.280728, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.1875 + } + }, + "oracle": { + "deliveredTokens": 1955, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.3125, + "recallAt10": 0.5625, + "precisionAt5": 1, + "precisionAt10": 0.9, + "reciprocalRank": 1, + "ndcgAt10": 0.974082, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5625 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3969, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.1875, + "recallAt10": 0.25, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.361702, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 3956, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.25, + "recallAt10": 0.5, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.788359, + "directRecallAt10": 0.833333, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 3969, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.1875, + "recallAt10": 0.25, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.361702, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3969, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.1875, + "recallAt10": 0.25, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.361702, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 3917, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.3125, + "recallAt10": 0.625, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.625 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7963, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.1875, + "recallAt10": 0.1875, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.280728, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.1875 + } + }, + "balancedV1": { + "deliveredTokens": 7997, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.25, + "recallAt10": 0.5, + "precisionAt5": 0.8, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.788359, + "directRecallAt10": 0.833333, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 7963, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.1875, + "recallAt10": 0.1875, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.280728, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.1875 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7963, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.1875, + "recallAt10": 0.1875, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.280728, + "directRecallAt10": 0.166667, + "supportingRecallAt10": 0.1875 + } + }, + "oracle": { + "deliveredTokens": 3917, + "metrics": { + "recallAt1": 0.0625, + "recallAt5": 0.3125, + "recallAt10": 0.625, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.625 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.437312, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-21379", + "repositorySequence": 62, + "positiveUnderStructuralOracle": true, + "directRecords": 2, + "supportingRecords": 11, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.153846, + "recallAt10": 0.307692, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.317337, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.307692 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.076923, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.042705, + "directRecallAt10": 0, + "supportingRecallAt10": 0.076923 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.153846, + "recallAt10": 0.307692, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.317337, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.307692 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.153846, + "recallAt10": 0.307692, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.317337, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.307692 + }, + "oracle": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.769231, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.769231 + } + }, + "volumes": { + "bm25Admitted": 3212, + "balancedV1": 3616, + "qarinahV2": 3212, + "qarinahV2NoGraph": 3212, + "oracle": 2273 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 492, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.076923, + "recallAt10": 0.076923, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.242497, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.076923 + } + }, + "balancedV1": { + "deliveredTokens": 435, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 492, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.076923, + "recallAt10": 0.076923, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.242497, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.076923 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 492, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.076923, + "recallAt10": 0.076923, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.242497, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.076923 + } + }, + "oracle": { + "deliveredTokens": 481, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.230769, + "recallAt10": 0.230769, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.690903, + "directRecallAt10": 1, + "supportingRecallAt10": 0.230769 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 999, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 988, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 999, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 999, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 936, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.384615, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.795642, + "directRecallAt10": 1, + "supportingRecallAt10": 0.384615 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1973, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.153846, + "recallAt10": 0.230769, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.287372, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.230769 + } + }, + "balancedV1": { + "deliveredTokens": 1993, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.076923, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.045636, + "directRecallAt10": 0, + "supportingRecallAt10": 0.076923 + } + }, + "qarinahV2": { + "deliveredTokens": 1973, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.153846, + "recallAt10": 0.230769, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.287372, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.230769 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1973, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.153846, + "recallAt10": 0.230769, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.287372, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.230769 + } + }, + "oracle": { + "deliveredTokens": 1899, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.615385, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.924399, + "directRecallAt10": 1, + "supportingRecallAt10": 0.615385 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3946, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.153846, + "recallAt10": 0.307692, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.317337, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.307692 + } + }, + "balancedV1": { + "deliveredTokens": 3985, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.076923, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.042705, + "directRecallAt10": 0, + "supportingRecallAt10": 0.076923 + } + }, + "qarinahV2": { + "deliveredTokens": 3946, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.153846, + "recallAt10": 0.307692, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.317337, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.307692 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3946, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.153846, + "recallAt10": 0.307692, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.317337, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.307692 + } + }, + "oracle": { + "deliveredTokens": 2726, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.769231, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.769231 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 8000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.153846, + "recallAt10": 0.307692, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.317337, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.307692 + } + }, + "balancedV1": { + "deliveredTokens": 7956, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.076923, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.042705, + "directRecallAt10": 0, + "supportingRecallAt10": 0.076923 + } + }, + "qarinahV2": { + "deliveredTokens": 8000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.153846, + "recallAt10": 0.307692, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.317337, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.307692 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 8000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.153846, + "recallAt10": 0.307692, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.317337, + "directRecallAt10": 0.5, + "supportingRecallAt10": 0.307692 + } + }, + "oracle": { + "deliveredTokens": 2726, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.769231, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.769231 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.402966, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-21612", + "repositorySequence": 63, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 17, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.058824, + "recallAt10": 0.235294, + "precisionAt5": 0.2, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.32413, + "directRecallAt10": null, + "supportingRecallAt10": 0.235294 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.176471, + "recallAt10": 0.235294, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.397192, + "directRecallAt10": null, + "supportingRecallAt10": 0.235294 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.058824, + "recallAt10": 0.235294, + "precisionAt5": 0.2, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.32413, + "directRecallAt10": null, + "supportingRecallAt10": 0.235294 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.058824, + "recallAt10": 0.235294, + "precisionAt5": 0.2, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.32413, + "directRecallAt10": null, + "supportingRecallAt10": 0.235294 + }, + "oracle": { + "recallAt1": 0.058824, + "recallAt5": 0.294118, + "recallAt10": 0.588235, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.588235 + } + }, + "volumes": { + "bm25Admitted": 3087, + "balancedV1": 2838, + "qarinahV2": 3087, + "qarinahV2NoGraph": 3087, + "oracle": 4482 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 438, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.058824, + "recallAt10": 0.058824, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.058824 + } + }, + "balancedV1": { + "deliveredTokens": 438, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.058824, + "recallAt10": 0.058824, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.058824 + } + }, + "qarinahV2": { + "deliveredTokens": 438, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.058824, + "recallAt10": 0.058824, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.058824 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 438, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.058824, + "recallAt10": 0.058824, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.138862, + "directRecallAt10": null, + "supportingRecallAt10": 0.058824 + } + }, + "oracle": { + "deliveredTokens": 454, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.176471, + "recallAt10": 0.176471, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.176471 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.058824, + "recallAt10": 0.058824, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.110046, + "directRecallAt10": null, + "supportingRecallAt10": 0.058824 + } + }, + "balancedV1": { + "deliveredTokens": 983, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.176471, + "recallAt10": 0.176471, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.318794, + "directRecallAt10": null, + "supportingRecallAt10": 0.176471 + } + }, + "qarinahV2": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.058824, + "recallAt10": 0.058824, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.110046, + "directRecallAt10": null, + "supportingRecallAt10": 0.058824 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 958, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.058824, + "recallAt10": 0.058824, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.110046, + "directRecallAt10": null, + "supportingRecallAt10": 0.058824 + } + }, + "oracle": { + "deliveredTokens": 913, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.176471, + "recallAt10": 0.176471, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.176471 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.117647, + "recallAt10": 0.117647, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.195189, + "directRecallAt10": null, + "supportingRecallAt10": 0.117647 + } + }, + "balancedV1": { + "deliveredTokens": 1918, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.176471, + "recallAt10": 0.235294, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.397192, + "directRecallAt10": null, + "supportingRecallAt10": 0.235294 + } + }, + "qarinahV2": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.117647, + "recallAt10": 0.117647, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.195189, + "directRecallAt10": null, + "supportingRecallAt10": 0.117647 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 2000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.117647, + "recallAt10": 0.117647, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.195189, + "directRecallAt10": null, + "supportingRecallAt10": 0.117647 + } + }, + "oracle": { + "deliveredTokens": 1927, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.294118, + "recallAt10": 0.411765, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.800694, + "directRecallAt10": null, + "supportingRecallAt10": 0.411765 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3963, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.058824, + "recallAt10": 0.235294, + "precisionAt5": 0.2, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.32413, + "directRecallAt10": null, + "supportingRecallAt10": 0.235294 + } + }, + "balancedV1": { + "deliveredTokens": 3988, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.176471, + "recallAt10": 0.235294, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.397192, + "directRecallAt10": null, + "supportingRecallAt10": 0.235294 + } + }, + "qarinahV2": { + "deliveredTokens": 3963, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.058824, + "recallAt10": 0.235294, + "precisionAt5": 0.2, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.32413, + "directRecallAt10": null, + "supportingRecallAt10": 0.235294 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3963, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.058824, + "recallAt10": 0.235294, + "precisionAt5": 0.2, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.32413, + "directRecallAt10": null, + "supportingRecallAt10": 0.235294 + } + }, + "oracle": { + "deliveredTokens": 3917, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.294118, + "recallAt10": 0.588235, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.588235 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7951, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.058824, + "recallAt10": 0.235294, + "precisionAt5": 0.2, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.32413, + "directRecallAt10": null, + "supportingRecallAt10": 0.235294 + } + }, + "balancedV1": { + "deliveredTokens": 7936, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.176471, + "recallAt10": 0.235294, + "precisionAt5": 0.6, + "precisionAt10": 0.4, + "reciprocalRank": 0.5, + "ndcgAt10": 0.397192, + "directRecallAt10": null, + "supportingRecallAt10": 0.235294 + } + }, + "qarinahV2": { + "deliveredTokens": 7951, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.058824, + "recallAt10": 0.235294, + "precisionAt5": 0.2, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.32413, + "directRecallAt10": null, + "supportingRecallAt10": 0.235294 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7951, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.058824, + "recallAt10": 0.235294, + "precisionAt5": 0.2, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.32413, + "directRecallAt10": null, + "supportingRecallAt10": 0.235294 + } + }, + "oracle": { + "deliveredTokens": 6082, + "metrics": { + "recallAt1": 0.058824, + "recallAt5": 0.294118, + "recallAt10": 0.588235, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.588235 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.4825, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-21614", + "repositorySequence": 64, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 13, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.076923, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.078398, + "directRecallAt10": null, + "supportingRecallAt10": 0.076923 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.076923, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.078398, + "directRecallAt10": null, + "supportingRecallAt10": 0.076923 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.076923, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.078398, + "directRecallAt10": null, + "supportingRecallAt10": 0.076923 + }, + "oracle": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.769231, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.769231 + } + }, + "volumes": { + "bm25Admitted": 2781, + "balancedV1": 3343, + "qarinahV2": 2781, + "qarinahV2NoGraph": 2781, + "oracle": 2590 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 417, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 512, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 417, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 417, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 505, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.076923, + "recallAt10": 0.076923, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.220092, + "directRecallAt10": null, + "supportingRecallAt10": 0.076923 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 976, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 992, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 976, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 976, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 960, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.230769, + "recallAt10": 0.230769, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.469, + "directRecallAt10": null, + "supportingRecallAt10": 0.230769 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1939, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.076923, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.078398, + "directRecallAt10": null, + "supportingRecallAt10": 0.076923 + } + }, + "balancedV1": { + "deliveredTokens": 1912, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1939, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.076923, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.078398, + "directRecallAt10": null, + "supportingRecallAt10": 0.076923 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1939, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.076923, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.078398, + "directRecallAt10": null, + "supportingRecallAt10": 0.076923 + } + }, + "oracle": { + "deliveredTokens": 1975, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.538462, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.800694, + "directRecallAt10": null, + "supportingRecallAt10": 0.538462 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3921, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.076923, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.078398, + "directRecallAt10": null, + "supportingRecallAt10": 0.076923 + } + }, + "balancedV1": { + "deliveredTokens": 3999, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3921, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.076923, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.078398, + "directRecallAt10": null, + "supportingRecallAt10": 0.076923 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3921, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.076923, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.078398, + "directRecallAt10": null, + "supportingRecallAt10": 0.076923 + } + }, + "oracle": { + "deliveredTokens": 3064, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.769231, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.769231 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.076923, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.078398, + "directRecallAt10": null, + "supportingRecallAt10": 0.076923 + } + }, + "balancedV1": { + "deliveredTokens": 8000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.076923, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.078398, + "directRecallAt10": null, + "supportingRecallAt10": 0.076923 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.076923, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.166667, + "ndcgAt10": 0.078398, + "directRecallAt10": null, + "supportingRecallAt10": 0.076923 + } + }, + "oracle": { + "deliveredTokens": 3064, + "metrics": { + "recallAt1": 0.076923, + "recallAt5": 0.384615, + "recallAt10": 0.769231, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 0.769231 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.445882, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-21627", + "repositorySequence": 65, + "positiveUnderStructuralOracle": true, + "directRecords": 3, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.262511, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.2 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.262511, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.2 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.262511, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.2 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.262511, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.2 + }, + "oracle": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4604, + "balancedV1": 3101, + "qarinahV2": 4604, + "qarinahV2NoGraph": 4604, + "oracle": 1346 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 505, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.41607, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.2 + } + }, + "balancedV1": { + "deliveredTokens": 472, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 505, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.41607, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 505, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.41607, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.2 + } + }, + "oracle": { + "deliveredTokens": 505, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.41607, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.2 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 932, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 977, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.262511, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2": { + "deliveredTokens": 932, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 932, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 971, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.6, + "recallAt10": 0.6, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.886617, + "directRecallAt10": 1, + "supportingRecallAt10": 0.6 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1985, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.262511, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.2 + } + }, + "balancedV1": { + "deliveredTokens": 1992, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.262511, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2": { + "deliveredTokens": 1985, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.262511, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1985, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.262511, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.2 + } + }, + "oracle": { + "deliveredTokens": 1346, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3963, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.262511, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.2 + } + }, + "balancedV1": { + "deliveredTokens": 3901, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.262511, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2": { + "deliveredTokens": 3963, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.262511, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3963, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.262511, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.2 + } + }, + "oracle": { + "deliveredTokens": 1346, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7968, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.262511, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.2 + } + }, + "balancedV1": { + "deliveredTokens": 7978, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.262511, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2": { + "deliveredTokens": 7968, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.262511, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7968, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.262511, + "directRecallAt10": 0.333333, + "supportingRecallAt10": 0.2 + } + }, + "oracle": { + "deliveredTokens": 1346, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.443333, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 5, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-21847", + "repositorySequence": 66, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 4, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.19519, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.19519, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.19519, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + }, + "oracle": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3261, + "balancedV1": 2578, + "qarinahV2": 3261, + "qarinahV2NoGraph": 3261, + "oracle": 1476 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 512, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.246302, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 471, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 512, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.246302, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 512, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.246302, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 384, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.636682, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 934, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.19519, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 953, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 934, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.19519, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 934, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.19519, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 956, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.831872, + "directRecallAt10": null, + "supportingRecallAt10": 0.75 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1949, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.19519, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 1922, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1949, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.19519, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1949, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.19519, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 1476, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3918, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.19519, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 3985, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3918, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.19519, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3918, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.19519, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 1476, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.19519, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 7946, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.19519, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.19519, + "directRecallAt10": null, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 1476, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.390132, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 4, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-22005", + "repositorySequence": 67, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "oracle": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3872, + "balancedV1": 5403, + "qarinahV2": 3872, + "qarinahV2NoGraph": 3872, + "oracle": 232 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 507, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 468, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 507, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 507, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 232, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 937, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 937, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 937, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 937, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 232, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1972, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1925, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1972, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1972, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 232, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 3947, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3994, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 232, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7953, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 7982, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7953, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7953, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 232, + "metrics": { + "recallAt1": 1, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.441129, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-22714", + "repositorySequence": 68, + "positiveUnderStructuralOracle": true, + "directRecords": 5, + "supportingRecords": 1, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.468991, + "directRecallAt10": 0.4, + "supportingRecallAt10": 0.5 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.166667, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.108677, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.166667 + }, + "qarinahV2": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.468991, + "directRecallAt10": 0.4, + "supportingRecallAt10": 0.5 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.468991, + "directRecallAt10": 0.4, + "supportingRecallAt10": 0.5 + }, + "oracle": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 6984, + "balancedV1": 3724, + "qarinahV2": 6984, + "qarinahV2NoGraph": 6984, + "oracle": 1525 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 505, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 491, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 486, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.694749, + "directRecallAt10": 0.6, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 936, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.531733, + "directRecallAt10": 0.4, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 949, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 936, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.531733, + "directRecallAt10": 0.4, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 936, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.531733, + "directRecallAt10": 0.4, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 877, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.694749, + "directRecallAt10": 0.6, + "supportingRecallAt10": 0.5 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1980, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.466445, + "directRecallAt10": 0.4, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 1955, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1980, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.466445, + "directRecallAt10": 0.4, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1980, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.466445, + "directRecallAt10": 0.4, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 1525, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3933, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.326031, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.166667 + } + }, + "balancedV1": { + "deliveredTokens": 3999, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.166667, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.108677, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2": { + "deliveredTokens": 3933, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.326031, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3933, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.326031, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.166667 + } + }, + "oracle": { + "deliveredTokens": 1525, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7998, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.468991, + "directRecallAt10": 0.4, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 7996, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.166667, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.142857, + "ndcgAt10": 0.108677, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2": { + "deliveredTokens": 7998, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.468991, + "directRecallAt10": 0.4, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7998, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.468991, + "directRecallAt10": 0.4, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 1525, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.524286, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-22840", + "repositorySequence": 69, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3177, + "balancedV1": 3663, + "qarinahV2": 3177, + "qarinahV2NoGraph": 3177, + "oracle": 871 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 510, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 493, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 339, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 980, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 998, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 871, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1933, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1900, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1933, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1933, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 871, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3903, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 3989, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3903, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3903, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 871, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7972, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 7926, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7972, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7972, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 871, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.422677, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-23117", + "repositorySequence": 70, + "positiveUnderStructuralOracle": true, + "directRecords": 5, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.4, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.334798, + "directRecallAt10": 0.4, + "supportingRecallAt10": 0.4 + }, + "balancedV1": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.33916, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.2 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.4, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.334798, + "directRecallAt10": 0.4, + "supportingRecallAt10": 0.4 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.4, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.334798, + "directRecallAt10": 0.4, + "supportingRecallAt10": 0.4 + }, + "oracle": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 5112, + "balancedV1": 4173, + "qarinahV2": 5112, + "qarinahV2NoGraph": 5112, + "oracle": 1212 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 475, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.33916, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.2 + } + }, + "balancedV1": { + "deliveredTokens": 475, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.33916, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2": { + "deliveredTokens": 475, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.33916, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 475, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.33916, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.2 + } + }, + "oracle": { + "deliveredTokens": 486, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.6, + "recallAt10": 0.6, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.722727, + "directRecallAt10": 0.6, + "supportingRecallAt10": 0.6 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 972, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 993, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.33916, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2": { + "deliveredTokens": 972, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 972, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 737, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.8, + "recallAt10": 0.8, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.868795, + "directRecallAt10": 0.8, + "supportingRecallAt10": 0.8 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.213986, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.2 + } + }, + "balancedV1": { + "deliveredTokens": 1971, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.33916, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2": { + "deliveredTokens": 1997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.213986, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.5, + "ndcgAt10": 0.213986, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.2 + } + }, + "oracle": { + "deliveredTokens": 1212, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3999, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.6, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.436895, + "directRecallAt10": 0.6, + "supportingRecallAt10": 0.6 + } + }, + "balancedV1": { + "deliveredTokens": 3969, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.33916, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2": { + "deliveredTokens": 3999, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.6, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.436895, + "directRecallAt10": 0.6, + "supportingRecallAt10": 0.6 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3999, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.6, + "precisionAt5": 0.2, + "precisionAt10": 0.3, + "reciprocalRank": 0.5, + "ndcgAt10": 0.436895, + "directRecallAt10": 0.6, + "supportingRecallAt10": 0.6 + } + }, + "oracle": { + "deliveredTokens": 1212, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.4, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.334798, + "directRecallAt10": 0.4, + "supportingRecallAt10": 0.4 + } + }, + "balancedV1": { + "deliveredTokens": 7911, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 0.2, + "recallAt10": 0.2, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.33916, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.2 + } + }, + "qarinahV2": { + "deliveredTokens": 7997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.4, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.334798, + "directRecallAt10": 0.4, + "supportingRecallAt10": 0.4 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.2, + "recallAt10": 0.4, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.5, + "ndcgAt10": 0.334798, + "directRecallAt10": 0.4, + "supportingRecallAt10": 0.4 + } + }, + "oracle": { + "deliveredTokens": 1212, + "metrics": { + "recallAt1": 0.2, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.552308, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-23191", + "repositorySequence": 71, + "positiveUnderStructuralOracle": true, + "directRecords": 5, + "supportingRecords": 13, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.213906, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.222222 + }, + "balancedV1": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.333333, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.568913, + "directRecallAt10": 0.6, + "supportingRecallAt10": 0.333333 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.213906, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.222222 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.213906, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.222222 + }, + "oracle": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.555556, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.555556 + } + }, + "volumes": { + "bm25Admitted": 2773, + "balancedV1": 3833, + "qarinahV2": 2773, + "qarinahV2NoGraph": 2773, + "oracle": 4767 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 474, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 512, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.299884, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2": { + "deliveredTokens": 474, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 474, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 434, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.612308, + "directRecallAt10": 0.6, + "supportingRecallAt10": 0.166667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 976, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.055556, + "recallAt10": 0.055556, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.047891, + "directRecallAt10": 0, + "supportingRecallAt10": 0.055556 + } + }, + "balancedV1": { + "deliveredTokens": 947, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.222222, + "recallAt10": 0.222222, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 0.423636, + "directRecallAt10": 0.4, + "supportingRecallAt10": 0.222222 + } + }, + "qarinahV2": { + "deliveredTokens": 976, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.055556, + "recallAt10": 0.055556, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.047891, + "directRecallAt10": 0, + "supportingRecallAt10": 0.055556 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 976, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.055556, + "recallAt10": 0.055556, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.047891, + "directRecallAt10": 0, + "supportingRecallAt10": 0.055556 + } + }, + "oracle": { + "deliveredTokens": 995, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.277778, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.773113, + "directRecallAt10": 0.8, + "supportingRecallAt10": 0.277778 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1972, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.166667, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.123259, + "directRecallAt10": 0, + "supportingRecallAt10": 0.166667 + } + }, + "balancedV1": { + "deliveredTokens": 1972, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.388889, + "precisionAt5": 1, + "precisionAt10": 0.7, + "reciprocalRank": 1, + "ndcgAt10": 0.664694, + "directRecallAt10": 0.8, + "supportingRecallAt10": 0.388889 + } + }, + "qarinahV2": { + "deliveredTokens": 1972, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.166667, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.123259, + "directRecallAt10": 0, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1972, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.166667, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.123259, + "directRecallAt10": 0, + "supportingRecallAt10": 0.166667 + } + }, + "oracle": { + "deliveredTokens": 1955, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.444444, + "precisionAt5": 1, + "precisionAt10": 0.8, + "reciprocalRank": 1, + "ndcgAt10": 0.94348, + "directRecallAt10": 1, + "supportingRecallAt10": 0.444444 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3934, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.213906, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.222222 + } + }, + "balancedV1": { + "deliveredTokens": 3947, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.333333, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.568913, + "directRecallAt10": 0.6, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 3934, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.213906, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.222222 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3934, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.213906, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.222222 + } + }, + "oracle": { + "deliveredTokens": 3991, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.555556, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.555556 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 8000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.213906, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.222222 + } + }, + "balancedV1": { + "deliveredTokens": 7935, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.333333, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 0.568913, + "directRecallAt10": 0.6, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2": { + "deliveredTokens": 8000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.213906, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.222222 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 8000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.111111, + "recallAt10": 0.222222, + "precisionAt5": 0.4, + "precisionAt10": 0.4, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.213906, + "directRecallAt10": 0.2, + "supportingRecallAt10": 0.222222 + } + }, + "oracle": { + "deliveredTokens": 6309, + "metrics": { + "recallAt1": 0.055556, + "recallAt5": 0.277778, + "recallAt10": 0.555556, + "precisionAt5": 1, + "precisionAt10": 1, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 0.555556 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "INSUFFICIENT_EVIDENCE", + "decision": "ABSTAIN", + "score": 0.396538, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "LOW_TERM_COVERAGE", + "INSUFFICIENT_THRESHOLD", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-23262", + "repositorySequence": 72, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.319071, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.319071, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.319071, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + }, + "oracle": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4153, + "balancedV1": 3864, + "qarinahV2": 4153, + "qarinahV2NoGraph": 4153, + "oracle": 1228 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 431, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 505, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 431, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 431, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 489, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.795976, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 889, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 949, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 889, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 889, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 680, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.75, + "recallAt10": 0.75, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.905587, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1982, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.254419, + "directRecallAt10": 1, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 1968, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1982, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.254419, + "directRecallAt10": 1, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1982, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.254419, + "directRecallAt10": 1, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 1228, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3901, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.319071, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 3978, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3901, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.319071, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3901, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.319071, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 1228, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.319071, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 7905, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.319071, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7991, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.319071, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 1228, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.630769, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 2, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-24066", + "repositorySequence": 73, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 2, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.193426, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + }, + "qarinahV2": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + }, + "oracle": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 3543, + "balancedV1": 3578, + "qarinahV2": 3543, + "qarinahV2NoGraph": 3543, + "oracle": 941 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 502, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 494, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 502, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 502, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 241, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 993, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 989, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 993, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 993, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.613147, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 941, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1993, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 1896, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1993, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1993, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 941, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3958, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 3934, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.193426, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 3958, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3958, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 941, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7964, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "balancedV1": { + "deliveredTokens": 7953, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0.5, + "precisionAt5": 0, + "precisionAt10": 0.1, + "reciprocalRank": 0.125, + "ndcgAt10": 0.193426, + "directRecallAt10": null, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 7964, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7964, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.919721, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "oracle": { + "deliveredTokens": 941, + "metrics": { + "recallAt1": 0.5, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.402647, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 3, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-24102", + "repositorySequence": 74, + "positiveUnderStructuralOracle": false, + "directRecords": 0, + "supportingRecords": 0, + "metrics": { + "bm25Admitted": null, + "balancedV1": null, + "qarinahV2": null, + "qarinahV2NoGraph": null, + "oracle": null + }, + "volumes": { + "bm25Admitted": 4525, + "balancedV1": 3867, + "qarinahV2": 4525, + "qarinahV2NoGraph": 4525, + "oracle": 0 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 475, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 475, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 475, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 475, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 950, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 995, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 950, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 950, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1948, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 1891, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 1948, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1948, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3996, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 3993, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 3996, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3996, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7967, + "metrics": null + }, + "balancedV1": { + "deliveredTokens": 7962, + "metrics": null + }, + "qarinahV2": { + "deliveredTokens": 7967, + "metrics": null + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7967, + "metrics": null + }, + "oracle": { + "deliveredTokens": 0, + "metrics": null + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.568701, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-24152", + "repositorySequence": 75, + "positiveUnderStructuralOracle": true, + "directRecords": 0, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + }, + "oracle": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 5781, + "balancedV1": 4607, + "qarinahV2": 5781, + "qarinahV2NoGraph": 5781, + "oracle": 1163 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 456, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 439, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 456, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 456, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 463, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 989, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1000, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 941, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 0.666667, + "recallAt10": 0.666667, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.765361, + "directRecallAt10": null, + "supportingRecallAt10": 0.666667 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1960, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 1997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 1960, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1960, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 1163, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 3960, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 3997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3997, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 1163, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7924, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 7971, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 7924, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7924, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": null, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 1163, + "metrics": { + "recallAt1": 0.333333, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": null, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.415, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-24213", + "repositorySequence": 76, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 3, + "metrics": { + "bm25Admitted": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.861968, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + }, + "balancedV1": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.327492, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + }, + "qarinahV2": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.861968, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + }, + "qarinahV2NoGraph": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.861968, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + }, + "oracle": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4256, + "balancedV1": 3760, + "qarinahV2": 4256, + "qarinahV2NoGraph": 4256, + "oracle": 1806 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 460, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.657663, + "directRecallAt10": 1, + "supportingRecallAt10": 0.25 + } + }, + "balancedV1": { + "deliveredTokens": 475, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 460, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.657663, + "directRecallAt10": 1, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 460, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.657663, + "directRecallAt10": 1, + "supportingRecallAt10": 0.25 + } + }, + "oracle": { + "deliveredTokens": 463, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.795976, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 922, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.795976, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 969, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2": { + "deliveredTokens": 922, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.795976, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 922, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.795976, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 463, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.795976, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1923, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.795976, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "balancedV1": { + "deliveredTokens": 1983, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.25, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 0.2, + "ndcgAt10": 0.254419, + "directRecallAt10": 1, + "supportingRecallAt10": 0.25 + } + }, + "qarinahV2": { + "deliveredTokens": 1923, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.795976, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1923, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.795976, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "oracle": { + "deliveredTokens": 1806, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3919, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.861968, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + }, + "balancedV1": { + "deliveredTokens": 3960, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.327492, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 3919, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.861968, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3919, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.861968, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + }, + "oracle": { + "deliveredTokens": 1806, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 8000, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.861968, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + }, + "balancedV1": { + "deliveredTokens": 7978, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.25, + "recallAt10": 0.5, + "precisionAt5": 0.2, + "precisionAt10": 0.2, + "reciprocalRank": 0.2, + "ndcgAt10": 0.327492, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 8000, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.861968, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 8000, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 0.5, + "recallAt10": 0.75, + "precisionAt5": 0.4, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.861968, + "directRecallAt10": 1, + "supportingRecallAt10": 0.75 + } + }, + "oracle": { + "deliveredTokens": 1806, + "metrics": { + "recallAt1": 0.25, + "recallAt5": 1, + "recallAt10": 1, + "precisionAt5": 0.8, + "precisionAt10": 0.4, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.61199, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + }, + { + "repository": "sympy/sympy", + "instanceId": "sympy__sympy-24909", + "repositorySequence": 77, + "positiveUnderStructuralOracle": true, + "directRecords": 1, + "supportingRecords": 5, + "metrics": { + "bm25Admitted": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.833333, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.370679, + "directRecallAt10": 0, + "supportingRecallAt10": 0.833333 + }, + "balancedV1": { + "recallAt1": 0.166667, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.505005, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + }, + "qarinahV2": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.833333, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.370679, + "directRecallAt10": 0, + "supportingRecallAt10": 0.833333 + }, + "qarinahV2NoGraph": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.833333, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.370679, + "directRecallAt10": 0, + "supportingRecallAt10": 0.833333 + }, + "oracle": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + }, + "volumes": { + "bm25Admitted": 4122, + "balancedV1": 2955, + "qarinahV2": 4122, + "qarinahV2NoGraph": 4122, + "oracle": 2680 + }, + "budgets": { + "512": { + "bm25Admitted": { + "deliveredTokens": 505, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 464, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.188513, + "directRecallAt10": 0, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2": { + "deliveredTokens": 505, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 505, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 463, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.307452, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + } + }, + "1000": { + "bm25Admitted": { + "deliveredTokens": 970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "balancedV1": { + "deliveredTokens": 972, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.166667, + "recallAt10": 0.166667, + "precisionAt5": 0.2, + "precisionAt10": 0.1, + "reciprocalRank": 1, + "ndcgAt10": 0.188513, + "directRecallAt10": 0, + "supportingRecallAt10": 0.166667 + } + }, + "qarinahV2": { + "deliveredTokens": 970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0, + "recallAt10": 0, + "precisionAt5": 0, + "precisionAt10": 0, + "reciprocalRank": 0, + "ndcgAt10": 0, + "directRecallAt10": 0, + "supportingRecallAt10": 0 + } + }, + "oracle": { + "deliveredTokens": 791, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 1, + "ndcgAt10": 0.684478, + "directRecallAt10": 1, + "supportingRecallAt10": 0.333333 + } + } + }, + "2000": { + "bm25Admitted": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.175445, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "balancedV1": { + "deliveredTokens": 1994, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.505005, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.175445, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 1970, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.333333, + "recallAt10": 0.333333, + "precisionAt5": 0.4, + "precisionAt10": 0.2, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.175445, + "directRecallAt10": 0, + "supportingRecallAt10": 0.333333 + } + }, + "oracle": { + "deliveredTokens": 1980, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 0.833333, + "precisionAt5": 1, + "precisionAt10": 0.5, + "reciprocalRank": 1, + "ndcgAt10": 0.93285, + "directRecallAt10": 1, + "supportingRecallAt10": 0.833333 + } + } + }, + "4000": { + "bm25Admitted": { + "deliveredTokens": 3909, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.833333, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.370679, + "directRecallAt10": 0, + "supportingRecallAt10": 0.833333 + } + }, + "balancedV1": { + "deliveredTokens": 3935, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.505005, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 3909, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.833333, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.370679, + "directRecallAt10": 0, + "supportingRecallAt10": 0.833333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 3909, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.833333, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.370679, + "directRecallAt10": 0, + "supportingRecallAt10": 0.833333 + } + }, + "oracle": { + "deliveredTokens": 2680, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + }, + "8000": { + "bm25Admitted": { + "deliveredTokens": 7932, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.833333, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.370679, + "directRecallAt10": 0, + "supportingRecallAt10": 0.833333 + } + }, + "balancedV1": { + "deliveredTokens": 7955, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.5, + "recallAt10": 0.5, + "precisionAt5": 0.6, + "precisionAt10": 0.3, + "reciprocalRank": 1, + "ndcgAt10": 0.505005, + "directRecallAt10": 1, + "supportingRecallAt10": 0.5 + } + }, + "qarinahV2": { + "deliveredTokens": 7932, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.833333, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.370679, + "directRecallAt10": 0, + "supportingRecallAt10": 0.833333 + } + }, + "qarinahV2NoGraph": { + "deliveredTokens": 7932, + "metrics": { + "recallAt1": 0, + "recallAt5": 0.5, + "recallAt10": 0.833333, + "precisionAt5": 0.6, + "precisionAt10": 0.5, + "reciprocalRank": 0.333333, + "ndcgAt10": 0.370679, + "directRecallAt10": 0, + "supportingRecallAt10": 0.833333 + } + }, + "oracle": { + "deliveredTokens": 2680, + "metrics": { + "recallAt1": 0.166667, + "recallAt5": 0.833333, + "recallAt10": 1, + "precisionAt5": 1, + "precisionAt10": 0.6, + "reciprocalRank": 1, + "ndcgAt10": 1, + "directRecallAt10": 1, + "supportingRecallAt10": 1 + } + } + } + }, + "evidenceSufficiency": { + "method": "evidence-sufficiency-v2", + "state": "PARTIALLY_SUPPORTED", + "decision": "ABSTAIN", + "score": 0.468571, + "directThreshold": 0.65, + "partialThreshold": 0.4, + "reasonCodes": [ + "PARTIAL_EVIDENCE_ONLY", + "ABSTAIN" + ] + }, + "noTemporalFutureItems": 1, + "noTemporalReturnedItems": 10 + } + ] + } + }, + "runtimeObservation": { + "node": "v24.15.0", + "platform": "win32-x64", + "totalEvaluationMs": 82071.2726 + } +} diff --git a/docs/API-REFERENCE.md b/docs/API-REFERENCE.md index b7909f5..185debc 100644 --- a/docs/API-REFERENCE.md +++ b/docs/API-REFERENCE.md @@ -18,7 +18,7 @@ import { captureClaudeHook } from "qarinah/claude"; import { createMcpServer, runMcpServer } from "qarinah/mcp"; ``` -The declarations shipped in `types/index.d.ts`, `types/codex.d.ts`, `types/claude.d.ts`, and `types/mcp.d.ts` are the exact compile-time contract for version 0.1.5. JSON Schemas are available through package exports such as `qarinah/schemas/event.json`. +The declarations shipped in `types/index.d.ts`, `types/codex.d.ts`, `types/claude.d.ts`, and `types/mcp.d.ts` are the exact compile-time contract for version 0.1.6. JSON Schemas are available through package exports such as `qarinah/schemas/event.json`. ## Runtime boundary @@ -61,9 +61,9 @@ Invalid JavaScript argument shapes generally throw `TypeError`. Storage, trust, ## Version and contract constants -| Export | Value in 0.1.5 | +| Export | Value in 0.1.6 | | --- | --- | -| `QARINAH_VERSION` | `"0.1.5"` | +| `QARINAH_VERSION` | `"0.1.6"` | | `EVENT_SCHEMA_VERSION` | `"qarinah.event.v1"` | | `CONTEXT_PACK_SCHEMA_VERSION` | `"qarinah.context-pack.v2"` | | `CONFIG_SCHEMA_VERSION` | `"qarinah.config.v1"` | @@ -437,7 +437,7 @@ function rankContextEvents( ): Readonly>; ``` -Runs local hybrid ranking after repository, time, retention, disclosure, and supersession admission. The default `admission-first-v2` profile preserves BM25 order for admissible lexical candidates, then fills from typo-tolerant and graph evidence; authority may promote an otherwise matched record. `balanced-v1` preserves the original reciprocal-rank-fusion and diversity behavior for reproducibility. `includeFuzzy` and `includeGraph` support explicit ablations. `temporalBoundary: "strict-before"` excludes records whose timestamp equals the query checkpoint; the default inclusive mode preserves normal as-of semantics. Every v2 result includes deterministic `evidence-sufficiency-v2` diagnostics. Only `DIRECTLY_SUPPORTED` produces `decision: "ACCEPT_DIRECT"`; both partial and insufficient evidence produce `decision: "ABSTAIN"`. At the 0.65 operating point the frozen development run observed zero direct false accepts, but exact 95% upper bounds remain 7.25% static and 11.22% online; this is not a universal semantic guarantee. Most callers should use `compileContext`, which also applies evidence gates and output budgets. +Runs local hybrid ranking after repository, time, retention, disclosure, and supersession admission. The default `admission-first-v2` profile preserves BM25 order for admissible lexical candidates, then fills from typo-tolerant and graph evidence; authority may promote an otherwise matched record. `balanced-v1` preserves the original reciprocal-rank-fusion and diversity behavior for reproducibility. `includeFuzzy` and `includeGraph` support explicit ablations. `temporalBoundary: "strict-before"` excludes records whose timestamp equals the query checkpoint; the default inclusive mode preserves normal as-of semantics. Every v2 result includes deterministic `evidence-sufficiency-v2` diagnostics. Only `DIRECTLY_SUPPORTED` produces `decision: "ACCEPT_DIRECT"`; both partial and insufficient evidence produce `decision: "ABSTAIN"`. At the 0.65 operating point the production-bound development-v0.4 recomputation observed 10/10 static and 15/15 online direct accepts as structural-oracle positives, with zero observed direct false accepts among 49 and 31 oracle-negative cases. Exact 95% false-acceptance upper bounds remain 7.25% static and 11.22% online; this is not a universal semantic guarantee. Most callers should use `compileContext`, which also applies evidence gates and output budgets. ## Context compilation diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index c917dc8..ebbdb93 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -14,17 +14,19 @@ The 98.75% and 89.05% values are intentionally different. They use the same 42-r All three results use the portable estimator `ceil(characters / 4)`. The exact committed fractions are `1 - 5,682 / 442,113`, `1 - 119 / 9,489`, and `1 - 1,039 / 9,489`. These are deterministic context-volume measurements, not provider usage receipts. -## Publication and evidence map +The 0.1.6 release evidence is bound in [`bench/results/benchmark-release-0.1.6.json`](../bench/results/benchmark-release-0.1.6.json) and checked by [`scripts/verify-benchmark-release-0.1.6.mjs`](../scripts/verify-benchmark-release-0.1.6.mjs). The manifest is a timestamped pre-publication verification receipt for the exact source later distributed as Qarinah 0.1.6, the website, and white-paper v1.3 with version DOI [`10.5281/zenodo.21843240`](https://doi.org/10.5281/zenodo.21843240). Its lifecycle fields intentionally preserve the state at local verification rather than acting as a live publication-status endpoint. It classifies the only provider-backed continuation receipt as historical 0.1.5 evidence. -Verified on 2026-08-07: +## Release evidence map -| Evidence | What it establishes | GitHub | npm 0.1.5 | qarinah.io | Zenodo paper v1.2 | -| --- | --- | --- | --- | --- | --- | -| Six-task software fixture | Estimated repeated-context volume and direct top-five coverage | Published | Published | Published | Published | -| Long-document fixture | Fixed-budget retrieval, supported-answer preservation, and unsupported-control rejection | Published | Published | Published | Published | -| Cross-session continuation fixture | Evidence-linked summary retrieval, a complete audit pack, and a compact model-facing capsule | Published | Published | Published | Published | -| SWE-bench Lite development study | Retrieval ranking, temporal leakage, repository isolation, and a conservative development gate on real repositories | Published | Published | Published | Published | -| SWE-bench Verified confirmatory study | Provider usage, patch resolution, cost, and human-rated quality | Protocol only | Protocol only | Protocol only | Protocol only; the study has not run | +Qarinah 0.1.6, the generated host integrations, the website, and white-paper v1.3 are released from one reviewed source commit. The evidence below is included with that release: + +| Evidence | What it establishes | Release status | +| --- | --- | --- | +| Six-task software fixture | Estimated repeated-context volume and direct top-five coverage | Included in Qarinah 0.1.6 and paper v1.3 | +| Long-document fixture | Fixed-budget retrieval, supported-answer preservation, and unsupported-control rejection | Included in Qarinah 0.1.6 and paper v1.3 | +| Cross-session continuation fixture | Evidence-linked summary retrieval, a complete audit pack, and a compact model-facing capsule | Included in Qarinah 0.1.6 and paper v1.3 | +| SWE-bench Lite development study | Retrieval ranking, temporal leakage, repository isolation, and the production-bound v0.4 gate on real repositories | Included in Qarinah 0.1.6 and paper v1.3 | +| SWE-bench Verified confirmatory study | Provider usage, patch resolution, cost, and human-rated quality | Protocol only; the study has not run | The SWE-bench work follows the official public dataset and evaluation framing, but the completed phase is a retrieval study rather than an official patch-resolution score. The frozen 40-task provider-backed study remains unexecuted, so Qarinah does not claim improved SWE-bench resolve rate or provider-native token savings. @@ -122,7 +124,7 @@ All three scales additionally verify: “Fail closed” therefore does not mean that long-document or multi-file retrieval failed. It means Qarinah refused to label unsupported evidence as direct. In this benchmark, **9 / 9 unsupported controls were successful fail-closed behavior**. -The fixture is synthetic so relevance and answers are completely auditable. It measures local retrieval, bounded evidence preservation, and projection integrity; it does not measure provider-reported tokens or coding-task completion. The context-volume values use portable `ceil(characters / 4)` estimates. See the [evaluator](../scripts/evaluate-multifile-context.mjs), [machine-readable result](../bench/results/multifile-context-0.1.5.json), and [artifact verifier](../scripts/verify-multifile-context.mjs). +The fixture is synthetic so relevance and answers are completely auditable. It measures local retrieval, bounded evidence preservation, and projection integrity; it does not measure provider-reported tokens or coding-task completion. The context-volume values use portable `ceil(characters / 4)` estimates. See the [evaluator](../scripts/evaluate-multifile-context.mjs), [machine-readable result](../bench/results/multifile-context-0.1.6.json), and [artifact verifier](../scripts/verify-multifile-context.mjs). ## Retrieval-regression fixture @@ -175,11 +177,11 @@ Keep these direct details beside the number: ## Cross-session continuation and evidence-linked summarization -Qarinah 0.1.5 ships a deterministic 42-record, two-session continuation fixture. A fresh logical session retrieves an inferred handoff summary at rank 3 after lifecycle capture deliberately makes the persisted read model stale. All three source event IDs and hashes remain embedded in the selected full pack, one raw source is also selected, the query leaves derived state byte-for-byte unchanged, and `doctor` passes. The complete cited audit pack uses 1,039 portable estimated tokens versus 9,489 for the full ledger, an 89.0505% estimated reduction. A separate 119-token model-facing handoff capsule points to that exact pack manifest and the selected summary event ID/hash, yielding a 98.7459% estimated reduction against the same unchanged history. The capsule is a compact projection, not a replacement for the complete evidence pack. +Qarinah 0.1.6 ships a deterministic 42-record, two-session continuation fixture. A fresh logical session retrieves an inferred handoff summary at rank 3 after lifecycle capture deliberately makes the persisted read model stale. All three source event IDs and hashes remain embedded in the selected full pack, one raw source is also selected, the query leaves derived state byte-for-byte unchanged, and `doctor` passes. The complete cited audit pack uses 1,039 portable estimated tokens versus 9,489 for the full ledger, an 89.0505% estimated reduction. A separate 119-token model-facing handoff capsule points to that exact pack manifest and the selected summary event ID/hash, yielding a 98.7459% estimated reduction against the same unchanged history. The capsule is a compact projection, not a replacement for the complete evidence pack. A separate authenticated Codex CLI smoke uses two distinct ephemeral sessions with native resume disabled. Session A diagnoses a failing fixture without editing; Session B must query Qarinah first, cite a retrieved event ID and hash, implement the fix, and pass the acceptance tests. The checked receipt retains hashes and normalized usage fields, not raw transcripts or local paths. -This is continuation-product evidence, not a controlled model-quality claim. See [the complete method and limitations](CROSS-SESSION-CONTINUATION-BENCHMARK.md), the [deterministic result](../bench/results/continuation-context-0.1.5.json), and the [provider-backed smoke receipt](../bench/results/codex-cross-session-continuation-0.1.5.json). +This is continuation-product evidence, not a controlled model-quality claim. See [the complete method and limitations](CROSS-SESSION-CONTINUATION-BENCHMARK.md), the [0.1.6 deterministic result](../bench/results/continuation-context-0.1.6.json), and the explicitly historical [0.1.5 provider-backed smoke receipt](../bench/results/codex-cross-session-continuation-0.1.5.json). No provider-backed 0.1.6 continuation receipt is claimed. ## Claims Qarinah does not make @@ -196,11 +198,13 @@ The research track follows the task framing introduced by Jimenez et al. in [SWE The frozen exploratory-v0.1 result is deliberately not a marketing headline. On the 79 held-out tasks with a prior production-file-overlap target, BM25 outperforms the original balanced-v1 Qarinah ranker: Recall@10 is 0.687 versus 0.518, and MRR is 0.430 versus 0.320. The no-temporal ablation returns 971 future citations, and the lexical coverage gate accepts all 161 tasks with no positive record under the file-overlap oracle. A separate 72-record governance suite returns zero forbidden records. -Development v0.2 is explicitly tuned after inspecting v0.1 and is not confirmatory. It resolves the official-page 11-repository statement against the 12-project revision artifact, adds a raw Parquet SHA-256, graded structural labels, static and online/prequential settings, fixed 512-8,000-token budgets, item/query leakage, calibration metrics, and repository-clustered bootstrap intervals. Admission-first Qarinah v2 exactly matches admitted BM25 ranking and improves online MRR over balanced-v1 from 0.601 to 0.696. Graph adds no measured ranking value. The original v0.2 interpretation counted partial evidence as accepted and produced a 90.32% false-acceptance rate among online tasks with no positive record under the structural oracle. +Development v0.2 is explicitly tuned after inspecting v0.1 and is not confirmatory. It resolves the official-page 11-repository statement against the 12-project revision artifact, adds a raw Parquet SHA-256, graded structural labels, static and online/prequential settings, fixed 512-8,000-token budgets, item/query leakage, calibration metrics, and repository-clustered bootstrap intervals. Admission-first Qarinah v2 exactly matches admitted BM25 ranking and improves online MRR over balanced-v1 from 0.601 to 0.696. Graph adds no measured ranking value. The frozen v0.2 artifact was produced by `evidence-sufficiency-v1`; its original interpretation counted partial evidence as accepted and produced a 90.32% false-acceptance rate among online tasks with no positive record under the structural oracle. Reproduction is bound to tag `research-retrieval-development-v0.2` at commit `bd566ac5ba7b302653b994fd0622d516fa74bbb8`, not mutable current source. + +Historical development v0.3 applies a conservative 0.65 direct threshold to those frozen v0.2 scores and treats partial evidence as abstention. It observed 8/8 static and 12/12 online accepted positives with 0/49 and 0/31 structural-oracle false accepts. Because the score source is v0.2, v0.3 is preserved as historical threshold calibration rather than described as current-product recomputation. -The conservative development-v0.3 gate accepts only direct evidence at the calibrated 0.65 threshold and treats partial evidence as abstention. At the frozen operating point it observed 0/49 direct false accepts in static evaluation and 0/31 online, including the leave-one-repository-out aggregation. Static direct precision is 8/8 with a 63.06%-100% exact 95% interval; online it is 12/12 with a 73.54%-100% interval. The false-acceptance intervals are 0%-7.25% and 0%-11.22%, respectively. Acceptance coverage is intentionally low at 3.33% static and 5.00% online. These are development structural-oracle results, not a universal semantic guarantee; a blinded 49-case relevance census is awaiting two independent human reviewers. +Development v0.4 recomputes the same inspected corpus with the current production `evidence-sufficiency-v2` implementation and binds the artifact to per-file source hashes. It accepts 10/240 static and 15/240 online queries as direct, all structural-oracle positives, while observing 0/49 and 0/31 direct false accepts. Direct precision is 10/10 with a 69.15%-100% exact 95% interval and 15/15 with a 78.20%-100% interval. False-acceptance intervals remain 0%-7.25% and 0%-11.22%; coverage remains deliberately low at 4.17% and 6.25%. These are development structural-oracle results, not a universal semantic guarantee; a blinded 49-case relevance census is awaiting two independent human reviewers. -See the complete [research protocol, limitations, repository citations, and results](RESEARCH-BENCHMARK.md). The committed artifacts preserve [`exploratory v0.1`](../bench/results/research-retrieval-0.1.2.json), [`development retrieval v0.2`](../bench/results/research-retrieval-development-v0.2.json), and [`conservative sufficiency v0.3`](../bench/results/research-sufficiency-development-v0.3.json). +See the complete [research protocol, limitations, repository citations, and results](RESEARCH-BENCHMARK.md). The committed artifacts preserve [`exploratory v0.1`](../bench/results/research-retrieval-0.1.2.json), [`historical development retrieval v0.2`](../bench/results/research-retrieval-development-v0.2.json), [`historical threshold calibration v0.3`](../bench/results/research-sufficiency-development-v0.3.json), and the [current production-bound development v0.4 recomputation](../bench/results/research-retrieval-development-v0.4.json). ## Next benchmark gate diff --git a/docs/CROSS-SESSION-CONTINUATION-BENCHMARK.md b/docs/CROSS-SESSION-CONTINUATION-BENCHMARK.md index badbc47..47fa172 100644 --- a/docs/CROSS-SESSION-CONTINUATION-BENCHMARK.md +++ b/docs/CROSS-SESSION-CONTINUATION-BENCHMARK.md @@ -1,6 +1,6 @@ # Evidence-linked cross-session continuation -Qarinah 0.1.5 ships a release-gated continuation benchmark and a provider-backed Codex product smoke test. Together they verify that a new coding-agent session can recover a compact handoff, inspect its source evidence, and continue work without native chat resume. +Qarinah 0.1.6 ships a release-gated deterministic continuation benchmark. A separate provider-backed Codex product smoke was recorded for Qarinah 0.1.5 and remains explicitly historical until a fresh authenticated 0.1.6 run is performed. Together they show that a new coding-agent session can recover a compact handoff, inspect its source evidence, and continue work without native chat resume, but the historical provider result is not evidence for the 0.1.6 release. These are product-readiness checks. The deterministic fixture is reproducible locally; the provider run is one authenticated smoke test. Neither is a substitute for the frozen multi-repository confirmatory study. @@ -34,14 +34,14 @@ After the persisted read model is built, the fixture appends Session B lifecycle The selected summary is marked `inferred`; its sources retain their original confidence. Only one raw source record needs to occupy the complete pack because all three source citations remain in the summary for on-demand inspection. The capsule is an additional bounded projection for model injection: it retains the untrusted-data label, summary event ID/hash, and complete-pack manifest hash while leaving the three raw source IDs/hashes in the auditable pack. The result reports partial lexical coverage and `DIRECTLY_SUPPORTED` evidence sufficiency; these are different diagnostics and neither claims that a model's eventual patch is correct. Token values use `ceil(characters / 4)` and are not provider billing data. -The evaluator and committed expected result are [`scripts/evaluate-continuation-context.mjs`](../scripts/evaluate-continuation-context.mjs) and [`bench/results/continuation-context-0.1.5.json`](../bench/results/continuation-context-0.1.5.json). +The evaluator and committed expected result are [`scripts/evaluate-continuation-context.mjs`](../scripts/evaluate-continuation-context.mjs) and [`bench/results/continuation-context-0.1.6.json`](../bench/results/continuation-context-0.1.6.json). ## Fresh Codex-to-Codex product smoke Run this separately from the deterministic release gate on a machine with an authenticated Codex CLI: ```sh -npm run smoke:codex-continuation -- --write +npm run smoke:codex-continuation:record npm run check:continuation-evidence ``` @@ -55,7 +55,7 @@ The runner creates a disposable Git repository with one failing immutable-releas 6. requires the final answer to cite a retrieved event ID and hash; 7. verifies the minimal fix, all acceptance tests, ledger integrity, distinct thread IDs, and zero transcript/path/credential leakage in the committed receipt. -The committed receipt is [`bench/results/codex-cross-session-continuation-0.1.5.json`](../bench/results/codex-cross-session-continuation-0.1.5.json). It stores hashes and normalized usage fields, not raw provider transcripts or local paths. The verifier rejects a wrong package version, changed implementation manifest, missing citations, reused session, failed test, failed doctor result, malformed receipt, or credential-like value. The implementation manifest hashes the normalized contents and paths of the shipped CLI, runtime, schemas, types, and Codex/Claude plugins, so it remains verifiable after a squash merge without relying on local Git history. +The historical receipt is [`bench/results/codex-cross-session-continuation-0.1.5.json`](../bench/results/codex-cross-session-continuation-0.1.5.json). It stores hashes and normalized usage fields, not raw provider transcripts or local paths. The verifier rejects changed recorded implementation evidence, missing citations, reused sessions, failed tests, failed doctor results, malformed receipts, or credential-like values. The implementation manifest hashes the normalized contents and paths of the shipped CLI, runtime, package metadata, schemas, types, and Codex/Claude plugins. `npm run check:continuation-evidence` verifies that receipt against exact Qarinah 0.1.5 release commit `69d5b899ad0b211134b53a7a1d21de079d975fd0` and reports `historical-0.1.5` scope instead of pretending the old provider run exercised 0.1.6. The check makes no provider call. A current receipt may be created only by actually running authenticated `npm run smoke:codex-continuation:record`; copying or synthesizing one is not valid evidence. ## Product fixes exercised by the benchmark diff --git a/docs/ECOSYSTEM-LAUNCH.md b/docs/ECOSYSTEM-LAUNCH.md index 5a4cc17..d6cd87d 100644 --- a/docs/ECOSYSTEM-LAUNCH.md +++ b/docs/ECOSYSTEM-LAUNCH.md @@ -26,7 +26,7 @@ Re-check registries and repositories immediately before publishing. As of the da | Maqam | `0.3.2` is the stable GitHub and npm release | Ready for demos that use only documented guarantees | | ProductLoop | `0.2.3` is the stable GitHub and npm release | Present as orchestration, not a second flagship launch | | Cockroach Crawler | `0.3.0` is the stable GitHub and npm `latest` release | Ready for bounded-crawl, provider-routing, optional reach, browser-host, and restricted serverless demos within documented limits | -| Qarinah | `0.1.0` stable release candidate | Publish only the reviewed source, package, paper, and website from one exact commit | +| Qarinah | `0.1.6` release | Publish only the reviewed source, package, paper, and website from one exact commit | Never describe a local candidate as published until the registry, tag, and public verification agree. diff --git a/docs/FAQ.md b/docs/FAQ.md index cdda718..5a0c679 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -140,7 +140,7 @@ Installing models or using external AI providers remains outside Qarinah's respo ## Which Node.js versions are supported? -Qarinah 0.1.5 supports maintained Node.js 22, 24, and 26 releases: +Qarinah 0.1.6 supports maintained Node.js 22, 24, and 26 releases: ```sh node --version @@ -286,7 +286,7 @@ npm run evaluate:context npm run benchmark ``` -See [BENCHMARKS.md](BENCHMARKS.md), [software-task results](../bench/results/software-task-context-0.1.1.json), [long-document results](../bench/results/long-document-context-0.1.1.json), and the [40/50/100-file project-context result](../bench/results/multifile-context-0.1.5.json). +See [BENCHMARKS.md](BENCHMARKS.md), [software-task results](../bench/results/software-task-context-0.1.1.json), [long-document results](../bench/results/long-document-context-0.1.1.json), and the [40/50/100-file project-context result](../bench/results/multifile-context-0.1.6.json). ## Where should I start? diff --git a/docs/GETTING-STARTED.md b/docs/GETTING-STARTED.md index f71e88a..e5c035b 100644 --- a/docs/GETTING-STARTED.md +++ b/docs/GETTING-STARTED.md @@ -76,7 +76,7 @@ The pack includes complete selected records, event IDs, hashes, a retrieval mani The default `admission-first-v2` profile first removes records that are out of repository, time, retention, disclosure, or supersession scope, then preserves BM25 order for the remaining lexical candidates. Typo-tolerant and graph matches may fill gaps, but cannot reintroduce rejected records. Research checkpoints should use `--temporal-boundary strict-before` so a record created at the task timestamp cannot leak into that task's query. -An additional experimental evidence-sufficiency diagnostic can be requested with `--minimum-evidence partial` or `--minimum-evidence direct`. In v2, partial evidence is an abstention; only the conservative direct state is accepted. The frozen development run observed zero direct false accepts under the structural oracle, but exact 95% upper bounds remain 7.25% static and 11.22% online and coverage is very low. Treat it as a workflow signal rather than proof that retrieved context is semantically sufficient. +An additional experimental evidence-sufficiency diagnostic can be requested with `--minimum-evidence partial` or `--minimum-evidence direct`. In v2, partial evidence is an abstention; only the conservative direct state is accepted. The production-bound development-v0.4 recomputation observed zero direct false accepts under the structural oracle among 49 static and 31 online negative cases, but exact 95% upper bounds remain 7.25% and 11.22% and direct coverage is only 4.17%-6.25%. Treat it as a workflow signal rather than proof that retrieved context is semantically sufficient. For agent callers, use JSON stdin: diff --git a/docs/HOST-INTEGRATIONS.md b/docs/HOST-INTEGRATIONS.md index f36d41e..b889efd 100644 --- a/docs/HOST-INTEGRATIONS.md +++ b/docs/HOST-INTEGRATIONS.md @@ -26,21 +26,21 @@ The host plugin and the project ledger have different scopes: Install the version-pinned stable release for personal use across projects: ```powershell -codex plugin marketplace add AjnasNB/qarinah --ref v0.1.5 +codex plugin marketplace add AjnasNB/qarinah --ref v0.1.6 codex plugin add qarinah@qarinah -claude plugin marketplace add AjnasNB/qarinah@v0.1.5 --scope user +claude plugin marketplace add AjnasNB/qarinah@v0.1.6 --scope user claude plugin install qarinah@qarinah --scope user ``` Claude Code also supports repository-shared `project` scope and gitignored per-user `local` scope: ```powershell -claude plugin marketplace add AjnasNB/qarinah@v0.1.5 --scope project +claude plugin marketplace add AjnasNB/qarinah@v0.1.6 --scope project claude plugin install qarinah@qarinah --scope project # Or keep the enablement personal to this repository. -claude plugin marketplace add AjnasNB/qarinah@v0.1.5 --scope local +claude plugin marketplace add AjnasNB/qarinah@v0.1.6 --scope local claude plugin install qarinah@qarinah --scope local ``` @@ -125,10 +125,10 @@ claude plugin install qarinah@qarinah --scope local The version-pinned install flow is: ```powershell -codex plugin marketplace add AjnasNB/qarinah --ref v0.1.5 +codex plugin marketplace add AjnasNB/qarinah --ref v0.1.6 codex plugin add qarinah@qarinah -claude plugin marketplace add AjnasNB/qarinah@v0.1.5 +claude plugin marketplace add AjnasNB/qarinah@v0.1.6 claude plugin install qarinah@qarinah ``` diff --git a/docs/MCP-GUIDE.md b/docs/MCP-GUIDE.md index 9ab9627..c9c17a6 100644 --- a/docs/MCP-GUIDE.md +++ b/docs/MCP-GUIDE.md @@ -1,6 +1,6 @@ # MCP guide -Qarinah 0.1.5 includes a native, zero-write Model Context Protocol server. It always provides local ledger status and integrity diagnostics. It provides `context.query` only after the user creates an explicit permit bound to the exact workspace, current consent-policy hash, and response ceilings. +Qarinah 0.1.6 includes a native, zero-write Model Context Protocol server. It always provides local ledger status and integrity diagnostics. It provides `context.query` only after the user creates an explicit permit bound to the exact workspace, current consent-policy hash, and response ceilings. That narrow boundary is intentional: @@ -15,7 +15,7 @@ That narrow boundary is intentional: | --- | --- | | MCP name | `io.github.AjnasNB/qarinah` | | npm package | `qarinah` | -| Version | `0.1.5` | +| Version | `0.1.6` | | Transport | `stdio` | | CLI entry | `npx qarinah mcp` | | Default tools | `context_status`, `context_doctor` | @@ -64,7 +64,7 @@ For a host configuration that accepts an MCP command object: "mcpServers": { "qarinah-context": { "command": "npx", - "args": ["-y", "qarinah@0.1.0", "mcp"] + "args": ["-y", "qarinah@0.1.6", "mcp"] } } } @@ -105,7 +105,7 @@ The packaged Codex integration defines: Install the reviewed release: ```sh -codex plugin marketplace add AjnasNB/qarinah --ref v0.1.0 +codex plugin marketplace add AjnasNB/qarinah --ref v0.1.6 codex plugin add qarinah@qarinah ``` @@ -147,14 +147,14 @@ The packaged Claude integration defines: Install: ```sh -claude plugin marketplace add AjnasNB/qarinah@v0.1.0 --scope user +claude plugin marketplace add AjnasNB/qarinah@v0.1.6 --scope user claude plugin install qarinah@qarinah --scope user ``` Project and local scopes are also supported by Claude Code: ```sh -claude plugin marketplace add AjnasNB/qarinah@v0.1.0 --scope project +claude plugin marketplace add AjnasNB/qarinah@v0.1.6 --scope project claude plugin install qarinah@qarinah --scope project ``` @@ -267,7 +267,7 @@ Compiles a bounded, cited context pack without writing to the workspace. This to Retrieval uses the `admission-first-v2` profile: repository, time, retention, disclosure, and supersession admission happens before ranking; admissible lexical candidates retain BM25 order, and fuzzy or graph evidence may only fill gaps. `temporalBoundary: "strict-before"` excludes evidence at the exact `asOf` timestamp and is recommended for prequential research evaluation. The default `inclusive` boundary is appropriate for normal as-of queries. -`minimumEvidence` accepts `any`, `partial`, or `direct`. A stricter value returns `CONTEXT_EVIDENCE_INSUFFICIENT` when the deterministic evidence diagnostic does not meet the requested level. In `evidence-sufficiency-v2`, only `DIRECTLY_SUPPORTED` produces `ACCEPT_DIRECT`; partial and insufficient evidence both produce `ABSTAIN`. The frozen development run observed zero direct false accepts at the conservative 0.65 operating point, but exact 95% upper bounds remain 7.25% static and 11.22% online and acceptance coverage is only 3.33%-5.00%. Callers must still inspect cited records and reason codes. +`minimumEvidence` accepts `any`, `partial`, or `direct`. A stricter value returns `CONTEXT_EVIDENCE_INSUFFICIENT` when the deterministic evidence diagnostic does not meet the requested level. In `evidence-sufficiency-v2`, only `DIRECTLY_SUPPORTED` produces `ACCEPT_DIRECT`; partial and insufficient evidence both produce `ABSTAIN`. The production-bound development-v0.4 recomputation observed zero direct false accepts at the conservative 0.65 operating point among 49 static and 31 online oracle-negative cases, but exact 95% upper bounds remain 7.25% and 11.22% and acceptance coverage is only 4.17%-6.25%. Callers must still inspect cited records and reason codes. ## Tool result errors diff --git a/docs/RECIPES.md b/docs/RECIPES.md index 82a0e9a..15b05b3 100644 --- a/docs/RECIPES.md +++ b/docs/RECIPES.md @@ -65,12 +65,12 @@ If direct coverage fails, inspect the error instead of silently relaxing the gat Install the reviewed host integrations: ```sh -codex plugin marketplace add AjnasNB/qarinah --ref v0.1.0 +codex plugin marketplace add AjnasNB/qarinah --ref v0.1.6 codex plugin add qarinah@qarinah ``` ```sh -claude plugin marketplace add AjnasNB/qarinah@v0.1.0 --scope user +claude plugin marketplace add AjnasNB/qarinah@v0.1.6 --scope user claude plugin install qarinah@qarinah --scope user ``` diff --git a/docs/RESEARCH-BENCHMARK.md b/docs/RESEARCH-BENCHMARK.md index 95a2f5b..a46c4e4 100644 --- a/docs/RESEARCH-BENCHMARK.md +++ b/docs/RESEARCH-BENCHMARK.md @@ -4,9 +4,9 @@ **Author:** Ajnas NB **Protocol version:** 1.0 -**Implementation:** Qarinah 0.1.2 +**Implementation:** Qarinah 0.1.6 **Date:** August 2026 -**Status:** Research draft; exploratory v0.1 frozen, development v0.2 preserved, conservative gate v0.3 completed, not peer-reviewed and not preregistered +**Status:** Research draft; exploratory v0.1 frozen, development v0.2 and historical calibration v0.3 preserved, current production-bound development v0.4 completed, not peer-reviewed and not preregistered **Artifact license:** Apache License 2.0; upstream repositories and benchmark data retain their own terms ## Abstract @@ -15,7 +15,7 @@ This research track studies whether evidence-linked, time-explicit project memor Exploratory v0.1 evaluates retrieval and governance, not patch generation. Only 79 of the 240 held-out tasks have a prior task that overlaps a gold production-file path, so retrieval effectiveness is reported on those 79 tasks and coverage behavior is reported on all 240. Plain BM25 outperforms the original Qarinah balanced hybrid ranking on this oracle: Recall@10 is 0.687 versus 0.518, and mean reciprocal rank is 0.430 versus 0.320. The lexical coverage gate accepts all 161 tasks for which the file-overlap oracle identifies no positive prior record; those tasks are not proven semantically unsupported. Removing time controls produces 971 future citations, or 42.44% of citations in that ablation. Separately, 72 constructed boundary records across all 12 repositories verify rejection of future, expired, stale, restricted, wrong-repository, and superseded evidence with zero forbidden records returned. -Exploratory v0.1 is frozen at Git tag `research-benchmark-exploratory-v0.1`. Development v0.2 changes Qarinah to admission-first lexical ranking and evaluates a deterministic graded structural oracle in both static and online/prequential settings. Qarinah v2 now matches admitted BM25 exactly on ranking while retaining governance controls and improves over the original balanced-v1 profile. Its graph stage adds no measured ranking value in this corpus. Development v0.3 changes the evidence decision to a conservative direct-only acceptance rule: partial evidence is exposed as an abstention, not accepted as sufficient. At this frozen operating point, we observed zero direct false accepts under the development structural oracle at the cost of accepting only 3.33% of static queries and 5.00% of online queries. The exact intervals remain wide, the underlying score remains poorly calibrated, and human relevance review is still pending. Provider-reported tokens, SWE-bench resolve rate, patch quality, model portability, cost, and independent human review remain unmeasured. +Exploratory v0.1 is frozen at Git tag `research-benchmark-exploratory-v0.1`. Development v0.2 changes Qarinah to admission-first lexical ranking and evaluates a deterministic graded structural oracle in both static and online/prequential settings. Qarinah v2 matches admitted BM25 exactly on ranking while retaining governance controls and improves over the original balanced-v1 profile. Its graph stage adds no measured ranking value in this corpus. Historical development v0.3 applies a conservative direct-only threshold to the frozen v0.2 `evidence-sufficiency-v1` scores. Development v0.4 separately recomputes current production `evidence-sufficiency-v2` behavior and binds the result to implementation hashes. V0.4 observes zero direct false accepts under the development structural oracle while accepting 4.17% of static queries and 6.25% online. The exact intervals remain wide, the underlying score remains poorly calibrated, and human relevance review is still pending. Provider-reported tokens, SWE-bench resolve rate, patch quality, model portability, cost, and independent human review remain unmeasured. ## 1. Research questions @@ -167,9 +167,9 @@ The experimental evidence-sufficiency score is not ready as a fail-closed semant The v2 no-temporal ablation returns 1,083 future items out of 2,288, a 47.33% item-level violation rate, and affects all 240 queries. This separates item-level from query-level leakage and reinforces that the higher-leakage ranking must not be treated as a valid system result. -### Conservative evidence decision v0.3 +### Historical conservative evidence decision v0.3 -Development v0.3 preserves the v0.2 score as a diagnostic but changes the decision boundary. A score of at least 0.65 returns `DIRECTLY_SUPPORTED` with decision `ACCEPT_DIRECT`; `PARTIALLY_SUPPORTED` always returns `ABSTAIN`. This distinction fixes the earlier mistake of counting partial evidence as a successful sufficiency decision. +Development v0.3 preserves the frozen v0.2 `evidence-sufficiency-v1` score as a diagnostic but changes the decision boundary. A score of at least 0.65 returns `DIRECTLY_SUPPORTED` with decision `ACCEPT_DIRECT`; `PARTIALLY_SUPPORTED` always returns `ABSTAIN`. This fixes the earlier interpretation that counted partial evidence as a successful sufficiency decision, but it is historical threshold calibration rather than a recomputation of current production `evidence-sufficiency-v2`. | Setting | No-positive tasks | Direct accepts | False accepts | Direct precision (exact 95% CI) | False-accept rate (exact 95% CI) | Direct recall | Coverage | | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | @@ -180,6 +180,17 @@ Leave-one-repository-out validation also observed zero direct false accepts, but The structural oracle itself remains unvalidated. A blinded audit artifact contains the complete census of all 49 static no-positive cases, with Qarinah scores, decisions, target gold patches, and target gold paths hidden. It is awaiting two independent human reviewers; reviewer agreement, Cohen's kappa, disagreements, and adjudicated labels must not be reported until real reviewers complete them. +### Current production-bound evidence decision v0.4 + +Development v0.4 reruns the same inspected 240-query corpus through current production `evidence-sufficiency-v2`. The artifact includes the exact method name, decisions, thresholds, reason codes, per-file source hashes, and a combined implementation digest. + +| Setting | No-positive tasks | Direct accepts | False accepts | Direct precision (exact 95% CI) | False-accept rate (exact 95% CI) | Direct recall | Coverage | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| Static | 49 | 10/240 | 0/49 | 100% (69.15%-100%) | 0% (0%-7.25%) | 5.24% | 4.17% | +| Online/prequential | 31 | 15/240 | 0/31 | 100% (78.20%-100%) | 0% (0%-11.22%) | 7.18% | 6.25% | + +The current product therefore retains the observed zero direct false accepts while accepting two additional static and three additional online structural-oracle positives relative to historical v0.3. This is still development evidence on the same inspected corpus. It does not establish semantic precision on new repositories, and it does not convert partial-score diagnostics into accepted evidence. + ## 8. What the pilot changes The research backlog is now evidence-driven: @@ -225,11 +236,13 @@ Until then, Qarinah must not claim improved SWE-bench resolve rate, provider-tok - Machine-readable result: [`bench/results/research-retrieval-0.1.2.json`](../bench/results/research-retrieval-0.1.2.json) - Frozen exploratory tag: `research-benchmark-exploratory-v0.1` - Development-v0.2 corpus: [`bench/research/swe-bench-lite-development-v0.2.json`](../bench/research/swe-bench-lite-development-v0.2.json) -- Development-v0.2 evaluator: [`scripts/evaluate-research-retrieval-v0.2.mjs`](../scripts/evaluate-research-retrieval-v0.2.mjs) +- Historical development-v0.2 evaluator guard and exact-tag instructions: [`scripts/evaluate-research-retrieval-v0.2.mjs`](../scripts/evaluate-research-retrieval-v0.2.mjs) - Development-v0.2 result: [`bench/results/research-retrieval-development-v0.2.json`](../bench/results/research-retrieval-development-v0.2.json) - Repository-count resolution: [`bench/research/repository-manifest-v0.2.json`](../bench/research/repository-manifest-v0.2.json) - Offline-backup receipt: [`bench/research/development-backup-v0.2.json`](../bench/research/development-backup-v0.2.json) -- Conservative sufficiency result: [`bench/results/research-sufficiency-development-v0.3.json`](../bench/results/research-sufficiency-development-v0.3.json) +- Historical conservative sufficiency result: [`bench/results/research-sufficiency-development-v0.3.json`](../bench/results/research-sufficiency-development-v0.3.json) +- Current production-bound v0.4 evaluator: [`scripts/evaluate-research-retrieval-v0.4.mjs`](../scripts/evaluate-research-retrieval-v0.4.mjs) +- Current production-bound v0.4 result: [`bench/results/research-retrieval-development-v0.4.json`](../bench/results/research-retrieval-development-v0.4.json) - Blinded relevance-review artifact: [`bench/research/relevance-audit-review-v0.3.json`](../bench/research/relevance-audit-review-v0.3.json) - Separate review-admin manifest: [`bench/research/relevance-audit-admin-v0.3.json`](../bench/research/relevance-audit-admin-v0.3.json) - Frozen final protocol receipt: [`bench/final/protocol-v1.json`](../bench/final/protocol-v1.json) @@ -241,7 +254,7 @@ Until then, Qarinah must not claim improved SWE-bench resolve rate, provider-tok - Pilot authorization guardrail: [`bench/final/pilot-authorization-v1.json`](../bench/final/pilot-authorization-v1.json) - Local execution-readiness receipt: [`bench/final/execution-readiness-v1.json`](../bench/final/execution-readiness-v1.json) -The evaluator fails if the pinned corpus, deterministic metrics, temporal leakage count, or governance results drift from the committed evidence. Runtime latency is retained as an observation and excluded from deterministic equality checks. +The current v0.4 evaluator fails if the pinned corpus, deterministic metrics, temporal leakage count, direct-decision counts, or bound implementation hashes drift from the committed evidence. Runtime latency is retained as an observation and excluded from deterministic equality checks. Historical v0.2 reproduction must use its exact tag and commit in a separate clean worktree. ## 11. Publication path diff --git a/docs/RESEARCH-DEVELOPMENT-RESULTS-v0.3.md b/docs/RESEARCH-DEVELOPMENT-RESULTS-v0.3.md index a9c6fdb..d80b386 100644 --- a/docs/RESEARCH-DEVELOPMENT-RESULTS-v0.3.md +++ b/docs/RESEARCH-DEVELOPMENT-RESULTS-v0.3.md @@ -2,6 +2,8 @@ Status: development evidence package, not peer reviewed and not a final confirmatory result. +Historical-version note: v0.3 applies a conservative 0.65 decision threshold to the frozen `evidence-sufficiency-v1` scores produced by development v0.2. It is preserved unchanged for research history and must not be described as a recomputation of the current production `evidence-sufficiency-v2` implementation. The production-bound recomputation is [development v0.4](RESEARCH-DEVELOPMENT-RESULTS-v0.4.md). + ## What changed Qarinah now separates retrieval from evidence sufficiency: diff --git a/docs/RESEARCH-DEVELOPMENT-RESULTS-v0.4.md b/docs/RESEARCH-DEVELOPMENT-RESULTS-v0.4.md new file mode 100644 index 0000000..f923066 --- /dev/null +++ b/docs/RESEARCH-DEVELOPMENT-RESULTS-v0.4.md @@ -0,0 +1,49 @@ +# Qarinah research-development result v0.4 + +Status: current-production recomputation on already inspected development data. This is not peer review, independent validation, or a final confirmatory result. + +## Why v0.4 exists + +The frozen v0.2 retrieval artifact was produced at tag `research-retrieval-development-v0.2`, commit `bd566ac5ba7b302653b994fd0622d516fa74bbb8`, with `evidence-sufficiency-v1`. Development v0.3 then applied the conservative 0.65 decision threshold to those frozen v0.2 scores. Both artifacts remain unchanged as historical development evidence. + +The product later shipped `evidence-sufficiency-v2`, including bounded code-entity extraction, 0.65/0.4 thresholds, and explicit `ACCEPT_DIRECT` versus `ABSTAIN` decisions. Re-running the old v0.2 evaluator against mutable current source therefore mixed experimental versions. The main-branch v0.2 command now fails fast and tells the operator to use the exact historical tag. Development v0.4 is the separate recomputation of the same 240-query corpus against the current production implementation. + +## Current production-bound result + +| Setting | Structural-oracle positives | Direct accepts | False accepts on oracle-negative queries | Direct precision (exact 95% CI) | Direct recall | Coverage | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Static | 191 / 240 | 10 / 240 | 0 / 49 | 100% (69.15%-100%) | 5.24% | 4.17% | +| Online/prequential | 209 / 240 | 15 / 240 | 0 / 31 | 100% (78.20%-100%) | 7.18% | 6.25% | + +The exact 95% upper bounds for the false-acceptance rate remain 7.25% static and 11.22% online. Zero observed errors on these oracle-negative cases is not proof of a zero population error rate. The structural oracle is based on file, symbol, and module overlap and has not been replaced by independent human relevance judgments. + +Partial evidence remains an abstention. If partial and direct states are combined only for score-diagnostic analysis, the structural-oracle negative rate is 55.10% static and 77.42% online. Those values are not production false accepts because only `DIRECTLY_SUPPORTED` maps to `ACCEPT_DIRECT`. + +Ranking is unchanged from v0.2: admission-first Qarinah exactly matches admitted BM25 on the shared admissible candidate set. Online Recall@10 is 0.5383, MRR is 0.6956, and the paired MRR improvement over balanced-v1 remains 0.0949 with a 12-repository clustered-bootstrap 95% interval of [0.0572, 0.1115]. Graph expansion again adds no ranking improvement on this workload. + +## Reproducibility + +Run the current production-bound evaluator without writing: + +```sh +npm run evaluate:research-retrieval:v0.4 +``` + +The committed artifact is [`bench/results/research-retrieval-development-v0.4.json`](../bench/results/research-retrieval-development-v0.4.json), SHA-256 `607359a947e7a849512d3fcb588bc88c2b34e1289f15b735a2de0c3895a21a18`. It records per-file implementation hashes and the combined implementation digest `sha256:034498c05fdf847078e639b6a2ed1efe2c72a1d714ca7f9bee337b3312bcd2eb`. `npm run check` recomputes v0.4 and fails if either measured values or bound production source hashes drift. + +To reproduce historical v0.2, use a separate clean worktree at the exact tag; do not run the old experiment through current source: + +```sh +git worktree add ../qarinah-research-v0.2 research-retrieval-development-v0.2 +cd ../qarinah-research-v0.2 +npm ci +npm run evaluate:research-retrieval:v0.2 +``` + +## Claim boundary + +- The corpus contains public SWE-bench Lite issues from real repositories, but it is development data that influenced the system. +- The relevance labels are a deterministic structural oracle, not independent semantic judgments. +- The run makes zero provider model calls, generates no patches, and does not execute the official SWE-bench Docker evaluator. +- It does not establish provider-token savings, task-resolution improvement, total-cost reduction, or universal evidence-gate precision. +- The separate Codex-to-Codex receipt remains one provider-backed synthetic product smoke. It is reproducibly verified by `npm run check:continuation-evidence` and can be freshly recorded only by actually running `npm run smoke:codex-continuation:record` with an authenticated Codex CLI. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 8ea1e04..1cbafbf 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -2,7 +2,7 @@ ## Implemented platform foundation -Qarinah 0.1.5 includes consent-gated MCP retrieval, one-command Codex/Claude/Cursor setup, a local memory dashboard, freshness checks, seven task packs, separate-authority multi-repository retrieval, optional reranking adapters, encrypted team bundles, role manifests, signed checkpoints, expanded quality evaluation, causal receipts across Cockroach, Qarinah, Maqam, execution, and observation, and a release-integrity gate that keeps package, runtime, type, MCP, website, paper, and benchmark metadata aligned. +Qarinah 0.1.6 includes consent-gated MCP retrieval, one-command Codex/Claude/Cursor setup, a local memory dashboard, freshness checks, seven task packs, separate-authority multi-repository retrieval, optional reranking adapters, encrypted team bundles, role manifests, signed checkpoints, expanded quality evaluation, causal receipts across Cockroach, Qarinah, Maqam, execution, and observation, and a release-integrity gate that keeps package, runtime, type, MCP, website, paper, and benchmark metadata aligned. The encrypted sync surface is a public self-hostable protocol foundation. A managed cross-device service, identity-provider federation, durable hosted transport, and organization administration remain future service work. diff --git a/docs/TOKEN-EFFICIENT-CONTEXT.md b/docs/TOKEN-EFFICIENT-CONTEXT.md index fb3eb50..5be7555 100644 --- a/docs/TOKEN-EFFICIENT-CONTEXT.md +++ b/docs/TOKEN-EFFICIENT-CONTEXT.md @@ -158,12 +158,12 @@ Qarinah stores memory in the opted-in project, not in one editor's private conve Install the reviewed plugins once per host: ```sh -codex plugin marketplace add AjnasNB/qarinah --ref v0.1.0 +codex plugin marketplace add AjnasNB/qarinah --ref v0.1.6 codex plugin add qarinah@qarinah ``` ```sh -claude plugin marketplace add AjnasNB/qarinah@v0.1.0 --scope user +claude plugin marketplace add AjnasNB/qarinah@v0.1.6 --scope user claude plugin install qarinah@qarinah --scope user ``` diff --git a/docs/WHITEPAPER.md b/docs/WHITEPAPER.md index d6bcb7d..af480e4 100644 --- a/docs/WHITEPAPER.md +++ b/docs/WHITEPAPER.md @@ -3,13 +3,15 @@ ## An evidence-linked project-memory compiler for coding agents **Author:** Ajnas NB
-**Paper version:** 1.2
-**Implementation:** Qarinah `0.1.5` stable open-source release
+**Paper version:** 1.3
+**Implementation:** Qarinah `0.1.6`
**Date:** August 2026
**License:** Apache License 2.0
-**Status:** Implementation-backed technical white paper for Qarinah 0.1.5. This version is not peer-reviewed. All measured claims identify their benchmark, denominator, estimator, and limits. +**Status:** Implementation-backed technical white paper for Qarinah 0.1.6. This version is not peer-reviewed. All measured claims identify their benchmark, denominator, estimator, and limits. Version DOI: [10.5281/zenodo.21843240](https://doi.org/10.5281/zenodo.21843240). Concept DOI: [10.5281/zenodo.21547684](https://doi.org/10.5281/zenodo.21547684). -[Download the publication PDF](https://github.com/AjnasNB/qarinah/blob/main/output/pdf/Qarinah-Technical-White-Paper-v1.2.pdf) +**Version note:** v1.3 incorporates the production-bound development-v0.4 recomputation. The published v1.2 PDF and its integrity receipt remain immutable historical artifacts. + +[Download the v1.3 PDF](https://github.com/AjnasNB/qarinah/blob/main/output/pdf/Qarinah-Technical-White-Paper-v1.3.pdf) > Qarinah turns permitted agent activity, project structure, decisions, approvals, and source evidence into a local, verifiable record. When a later task needs context, Qarinah compiles a small cited pack instead of replaying the complete project history. @@ -27,7 +29,7 @@ The implementation is local-first, model-agnostic, and explicit about capture. A The committed software-task evaluation compares full-history replay with Qarinah packs while keeping the same current-task source material on both sides. Across 240 retained records and six software scenarios, the measured context falls from 442,113 to 5,682 estimated input-context tokens: a 98.7148% reduction and a 77.81:1 compression ratio. Every required target ranks in the top five with direct evidence coverage, and no model-written summaries are used. A separate 42-record, two-session continuation fixture measures two outputs against the same 9,489-token history: a 119-token model-facing capsule (98.7459% reduction) and a 1,039-token complete cited audit pack (89.0505% reduction). The capsule retains the selected summary identity and audit-pack manifest pointer; the larger pack retains all summary-source identities and hashes. -A development-stage real-repository study uses the official public SWE-bench Lite task artifact and the software-issue framing introduced by Jimenez et al. [1]. It orders each repository chronologically, uses 60 early tasks to build memory, and evaluates 240 later queries. Admission-first Qarinah v2 matches admitted BM25 ranking, while its temporal and authority filters prevent the future and forbidden evidence admitted by ablations. Online mean reciprocal rank improves from 0.6007 for the earlier balanced profile to 0.6956 for v2; the repository-clustered 95% interval for the paired difference is [0.0572, 0.1115]. Graph expansion adds no ranking improvement in this corpus. A conservative development gate observes zero direct false accepts under the structural oracle but accepts only 3.33% of static and 5.00% of online queries, so it is not presented as a universal semantic guarantee. +A development-stage real-repository study uses the official public SWE-bench Lite task artifact and the software-issue framing introduced by Jimenez et al. [1]. It orders each repository chronologically, uses 60 early tasks to build memory, and evaluates 240 later queries. Admission-first Qarinah v2 matches admitted BM25 ranking, while its temporal and authority filters prevent the future and forbidden evidence admitted by ablations. Online mean reciprocal rank improves from 0.6007 for the earlier balanced profile to 0.6956 for v2; the repository-clustered 95% interval for the paired difference is [0.0572, 0.1115]. Graph expansion adds no ranking improvement in this corpus. The production-bound development-v0.4 recomputation observes zero direct false accepts under the structural oracle while accepting 4.17% of static and 6.25% of online queries, so it is not presented as a universal semantic guarantee. A separate deterministic scale regression builds 40-, 50-, and 100-file repositories containing nested JavaScript and Markdown, resolved relationships, lexical distractors, graph-only evidence, supersession, contradiction, and deliberately stale projections. All 380 exact and typo-tolerant positive queries return the correct cited record at rank 1 and preserve its answer. All 190 exact queries use SQLite and are accepted as direct; all 190 typo queries retrieve the correct evidence but conservatively abstain; and all nine unsupported controls are rejected. The worst-case estimated context reductions at the three scales are 93.4803%, 94.6948%, and 97.1521%. This is synthetic local retrieval and projection-integrity evidence, not provider-billed usage or coding-task success. @@ -620,7 +622,7 @@ The development study follows the real GitHub issue-resolution framing introduce Within each repository, tasks are ordered by creation time. The earliest 20% (60 tasks total) build prior memory; the remaining 240 tasks are queried only against evidence available strictly before their timestamp. Static evaluation freezes the warm-up memory. Online/prequential evaluation admits an earlier held-out task only after its query has been scored. This prevents future-task evidence from entering an earlier query. -Compared retrieval conditions include admitted BM25, the earlier balanced Qarinah profile, Qarinah v2, Qarinah v2 without graph expansion, and a structural oracle. BM25 is treated as a strong lexical baseline following the Okapi family [3]. The temporal, update, and abstention categories are informed by LongMemEval [4], extended here with repository isolation, evidence identity, supersession, retention, and disclosure authority. +Compared retrieval conditions include admitted BM25, the earlier balanced Qarinah profile, Qarinah v2, Qarinah v2 without graph expansion, and a structural oracle. BM25 is treated as a strong lexical baseline following the Okapi family [3]. The temporal, update, and abstention categories are informed by LongMemEval [4], extended here with repository isolation, evidence identity, supersession, retention, and disclosure authority. Historical v0.2 and v0.3 artifacts retain their original score provenance; development v0.4 separately recomputes current `evidence-sufficiency-v2` decisions and records per-file implementation hashes. This phase makes zero provider model calls and does not apply generated patches. Labels are a deterministic graded structural oracle based on pre-task file, symbol, API, test, and error overlap. They are not independent human relevance judgments. Results are development evidence because v2 and the conservative evidence gate were designed after v0.1 was inspected. The next protocol freezes 40 paired tasks from the 500-instance, human-validated SWE-bench Verified corpus [5], excluding Lite development instances before any final outcome is observed. @@ -709,13 +711,13 @@ The fresh session retrieves the inferred handoff summary at rank 3. All three su | Qarinah v2 Recall@10 | 0.7626 | 0.5383 | | Qarinah v2 MRR | 0.7032 | 0.6956 | | Earlier balanced-profile MRR | 0.6443 | 0.6007 | -| Qarinah v2 direct-evidence acceptance | 8 / 240 | 12 / 240 | +| Qarinah v2 direct-evidence acceptance | 10 / 240 | 15 / 240 | | Observed direct false accepts under structural oracle | 0 / 49 | 0 / 31 | -| Acceptance coverage | 3.33% | 5.00% | +| Acceptance coverage | 4.17% | 6.25% | Qarinah v2 exactly matches admitted BM25 ranking in both settings. Its contribution in this experiment is therefore not a new ranking algorithm; it is lexical ranking inside temporal, repository, retention, supersession, and authority admission boundaries. Removing graph expansion produces identical ranking metrics, so graph retrieval adds no measured ranking value in this corpus. The graph remains useful for provenance, conflict, supersession, and relationship representation. -For the online paired comparison with the earlier balanced profile, the MRR difference is 0.0949 with a repository-clustered 95% bootstrap interval of [0.0572, 0.1115]. The Recall@10 difference is 0.0649 with interval [-0.0150, 0.1025], so improved total relevant-record recall is not established. The conservative evidence gate observes zero direct false accepts under the development structural oracle, but exact 95% upper bounds remain 7.25% static and 11.22% online, and the gate abstains on most queries. These results support a small high-confidence subset, not a claim of perfect evidence detection. +For the online paired comparison with the earlier balanced profile, the MRR difference is 0.0949 with a repository-clustered 95% bootstrap interval of [0.0572, 0.1115]. The Recall@10 difference is 0.0649 with interval [-0.0150, 0.1025], so improved total relevant-record recall is not established. The production-bound v0.4 gate observes zero direct false accepts under the development structural oracle, but exact 95% upper bounds remain 7.25% static and 11.22% online, and the gate abstains on most queries. Direct-precision intervals are 69.15%-100% static and 78.20%-100% online. These results support a small high-confidence subset, not a claim of perfect evidence detection. ### 15.4 Long-document retrieval @@ -803,7 +805,7 @@ npm run evaluate:continuation npm run evaluate:long-document npm run evaluate:multifile-context npm run evaluate:context -npm run evaluate:research-retrieval:v0.2 +npm run evaluate:research-retrieval:v0.4 npm run evaluate:research-sufficiency:v0.3 npm run check:benchmark-release npm run benchmark @@ -813,19 +815,20 @@ The relevant evidence is committed at: - [`bench/fixtures/software-task-scenarios.mjs`](../bench/fixtures/software-task-scenarios.mjs); - [`bench/results/software-task-context-0.1.0.json`](../bench/results/software-task-context-0.1.0.json); -- [`bench/results/continuation-context-0.1.5.json`](../bench/results/continuation-context-0.1.5.json); +- [`bench/results/continuation-context-0.1.6.json`](../bench/results/continuation-context-0.1.6.json); - [`bench/results/long-document-context-0.1.0.json`](../bench/results/long-document-context-0.1.0.json); -- [`bench/results/multifile-context-0.1.5.json`](../bench/results/multifile-context-0.1.5.json); +- [`bench/results/multifile-context-0.1.6.json`](../bench/results/multifile-context-0.1.6.json); - [`bench/results/context-evaluation-0.1.0.json`](../bench/results/context-evaluation-0.1.0.json); - [`bench/results/research-retrieval-development-v0.2.json`](../bench/results/research-retrieval-development-v0.2.json); - [`bench/results/research-sufficiency-development-v0.3.json`](../bench/results/research-sufficiency-development-v0.3.json); -- [`bench/results/benchmark-release-0.1.5.json`](../bench/results/benchmark-release-0.1.5.json); +- [`bench/results/research-retrieval-development-v0.4.json`](../bench/results/research-retrieval-development-v0.4.json); +- [`bench/results/benchmark-release-0.1.6.json`](../bench/results/benchmark-release-0.1.6.json); - [`scripts/evaluate-software-tasks.mjs`](../scripts/evaluate-software-tasks.mjs); - [`scripts/evaluate-long-document.mjs`](../scripts/evaluate-long-document.mjs); - [`scripts/evaluate-multifile-context.mjs`](../scripts/evaluate-multifile-context.mjs); - [`scripts/evaluate-context.mjs`](../scripts/evaluate-context.mjs); and - [`scripts/verify-benchmark-evidence.mjs`](../scripts/verify-benchmark-evidence.mjs); and -- [`scripts/verify-benchmark-release-0.1.5.mjs`](../scripts/verify-benchmark-release-0.1.5.mjs). +- [`scripts/verify-benchmark-release-0.1.6.mjs`](../scripts/verify-benchmark-release-0.1.6.mjs). The evidence verifier recomputes public percentages and rejects unsupported headline claims. Documentation checks verify local links, architecture-source integrity, and encoding. The complete package check also rebuilds both host plugins, exercises MCP transport, runs the test and type matrices, executes every benchmark, and performs a dry-run package build. @@ -863,8 +866,10 @@ Recommended publication metadata: - **Title:** *Qarinah: Less Context. More Proof.* - **Subtitle:** *An evidence-linked project-memory compiler for coding agents* - **Author:** Ajnas NB -- **Implementation version:** `0.1.5` -- **Paper version:** 1.2 +- **Implementation version:** `0.1.6` +- **Paper version:** 1.3 +- **Version DOI:** `10.5281/zenodo.21843240` +- **Concept DOI:** `10.5281/zenodo.21547684` - **License:** Apache-2.0 - **Canonical source:** this repository at one reviewed commit - **Evidence:** the machine-readable files and commands listed in Section 17 @@ -905,7 +910,7 @@ Coding agents need continuity, but continuity should not require replaying every Qarinah keeps durable evidence and task-time context as two different artifacts. The append-only ledger preserves the permitted project record. Deterministic projections make that record searchable and inspectable. The context compiler selects a small cited working set under an explicit budget. Conflicts, supersession, authority, retention, and evidence coverage remain visible rather than being compressed away. -Qarinah 0.1.5 demonstrates that this design works end to end across local storage, SQLite retrieval, graph and Markdown projections, project structure, Codex and Claude Code adapters, MCP diagnostics, Maqam governance, crawler evidence, workflow provenance, and portable OKF export. Its committed evaluations show large context-volume reductions while preserving the required evidence in the evaluated scenarios. The 380-query multi-file regression extends that evidence through 100-file deterministic projects, while the real-repository development study identifies both useful controls and unresolved limits. +Qarinah 0.1.6 demonstrates that this design works end to end across local storage, SQLite retrieval, graph and Markdown projections, project structure, Codex and Claude Code adapters, MCP diagnostics, Maqam governance, crawler evidence, workflow provenance, and portable OKF export. Its committed evaluations show large context-volume reductions while preserving the required evidence in the evaluated scenarios. The 380-query multi-file regression extends that evidence through 100-file deterministic projects, while the production-bound v0.4 real-repository development study identifies both useful controls and unresolved limits. The central promise is intentionally simple: @@ -941,7 +946,7 @@ The author gratefully acknowledges Shahin Ahammed, Qarinah's non-technical cofou | 9 / 9 multi-file unsupported controls rejected | Direct-coverage controls across all three workspaces | Correct abstention; does not mean supported long-document or project retrieval failed | | 93.4803%-97.1521% multi-file worst-case estimated reduction | Largest bounded pack versus each generated ledger | Portable character estimator, not provider billing | | Qarinah v2 matches admitted BM25 | SWE-bench Lite development retrieval result | Ranking equality; Qarinah retains temporal, repository, retention, supersession, and authority admission | -| Zero observed direct false accepts at the conservative gate | 0/49 static and 0/31 online under the structural development oracle | Low 3.33%-5.00% coverage, wide exact intervals, and no independent human relevance labels | +| Zero observed direct false accepts at the production-bound v0.4 gate | 0/49 static and 0/31 online under the structural development oracle | Low 4.17%-6.25% coverage, wide exact intervals, and no independent human relevance labels | | Local-first and no Qarinah API key | Runtime architecture and package dependencies | AI hosts and external sources may still require their own access | | Deterministic rebuildable graph, index, Markdown, and OKF | Source implementation and test suite | Reproducibility depends on the same verified event head and build inputs | @@ -963,6 +968,7 @@ The author gratefully acknowledges Shahin Ahammed, Qarinah's non-technical cofou ```text Ajnas NB. "Qarinah: Less Context. More Proof. An evidence-linked project-memory compiler for coding agents." Technical white paper, -version 1.2, August 2026. Qarinah 0.1.5. Concept DOI: +version 1.3, August 2026. Qarinah 0.1.6. Version DOI: +https://doi.org/10.5281/zenodo.21843240. Concept DOI: https://doi.org/10.5281/zenodo.21547684. ``` diff --git a/integrations/claude/qarinah/.claude-plugin/plugin.json b/integrations/claude/qarinah/.claude-plugin/plugin.json index 5dc8974..551abae 100644 --- a/integrations/claude/qarinah/.claude-plugin/plugin.json +++ b/integrations/claude/qarinah/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "qarinah", - "version": "0.1.5", + "version": "0.1.6", "description": "Qarinah is evidence-linked project memory for coding agents - compact enough to save context, inspectable enough to trust.", "author": { "name": "Ajnas NB", diff --git a/integrations/claude/qarinah/runtime/qarinah.mjs b/integrations/claude/qarinah/runtime/qarinah.mjs index fffc6f2..42aefb0 100644 --- a/integrations/claude/qarinah/runtime/qarinah.mjs +++ b/integrations/claude/qarinah/runtime/qarinah.mjs @@ -1,5 +1,5 @@ #!/usr/bin/env node -// GENERATED by scripts/build-plugins.mjs for Qarinah 0.1.5. DO NOT EDIT. +// GENERATED by scripts/build-plugins.mjs for Qarinah 0.1.6. DO NOT EDIT. var __create = Object.create; var __defProp = Object.defineProperty; var __getOwnPropDesc = Object.getOwnPropertyDescriptor; @@ -2948,7 +2948,7 @@ import process2 from "node:process"; init_errors(); // src/version.js -var QARINAH_VERSION = "0.1.5"; +var QARINAH_VERSION = "0.1.6"; // src/index.js init_store(); diff --git a/integrations/codex/qarinah/.codex-plugin/plugin.json b/integrations/codex/qarinah/.codex-plugin/plugin.json index 89d402e..a70c541 100644 --- a/integrations/codex/qarinah/.codex-plugin/plugin.json +++ b/integrations/codex/qarinah/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "qarinah", - "version": "0.1.5", + "version": "0.1.6", "description": "Qarinah is evidence-linked project memory for coding agents - compact enough to save context, inspectable enough to trust.", "author": { "name": "Ajnas NB", diff --git a/integrations/codex/qarinah/runtime/qarinah.mjs b/integrations/codex/qarinah/runtime/qarinah.mjs index fffc6f2..42aefb0 100644 --- a/integrations/codex/qarinah/runtime/qarinah.mjs +++ b/integrations/codex/qarinah/runtime/qarinah.mjs @@ -1,5 +1,5 @@ #!/usr/bin/env node -// GENERATED by scripts/build-plugins.mjs for Qarinah 0.1.5. DO NOT EDIT. +// GENERATED by scripts/build-plugins.mjs for Qarinah 0.1.6. DO NOT EDIT. var __create = Object.create; var __defProp = Object.defineProperty; var __getOwnPropDesc = Object.getOwnPropertyDescriptor; @@ -2948,7 +2948,7 @@ import process2 from "node:process"; init_errors(); // src/version.js -var QARINAH_VERSION = "0.1.5"; +var QARINAH_VERSION = "0.1.6"; // src/index.js init_store(); diff --git a/output/pdf/Qarinah-Technical-White-Paper-v1.3.build.json b/output/pdf/Qarinah-Technical-White-Paper-v1.3.build.json new file mode 100644 index 0000000..1e78546 --- /dev/null +++ b/output/pdf/Qarinah-Technical-White-Paper-v1.3.build.json @@ -0,0 +1,53 @@ +{ + "combinedSourceSha256": "sha256:f12b901d8409e6ffe68a1f9aebd8d58968d7dee248ac9f6fe1e93f268209ab82", + "generator": { + "command": "python scripts/build-whitepaper-pdf-v1.3.py", + "fonts": [ + { + "fileName": "arial.ttf", + "registeredName": "QBody", + "role": "body", + "sha256": "b3658eadae55e682b5f69eb64c439c1ecc8f196c0bb8d4756d145d13bc86476a" + }, + { + "fileName": "arialbd.ttf", + "registeredName": "QBody-Bold", + "role": "bold", + "sha256": "e8f4e3baf6cc35fed6fcce3a540e8b39e8f6cda1d22a28f2ec8f526fef7a43f5" + }, + { + "fileName": "ariali.ttf", + "registeredName": "QBody-Italic", + "role": "italic", + "sha256": "86b32db9a06f9694e2a3760c42e5117bcdc5cc1255bb5186ca8ce0305e22f288" + }, + { + "fileName": "consola.ttf", + "registeredName": "QMono", + "role": "monospace", + "sha256": "cf00b507b3286870cc5064ebd0633c303f70b491a4af25eec2d32df413db0179" + } + ], + "platform": "win32", + "pythonImplementation": "CPython", + "pythonVersion": "3.12.13", + "reportlabVersion": "4.4.9" + }, + "paperVersion": "1.3", + "schemaVersion": "qarinah.white-paper-build.v1", + "sourceDigestAlgorithm": "sha256(file-bytes + NUL + file-bytes + NUL + file-bytes; listed order)", + "sources": [ + { + "path": "docs/WHITEPAPER.md", + "sha256": "sha256:de7da69087eaf2e9bbad40122f724cff488834aa366d5205b14ca58a9b75e840" + }, + { + "path": "scripts/build-whitepaper-pdf.py", + "sha256": "sha256:bab8180794f7d5416c06d394a215ea3b85cc5520073fd1f860e1bea169155bdf" + }, + { + "path": "scripts/build-whitepaper-pdf-v1.3.py", + "sha256": "sha256:f659d02aa1dbffae269a745d6917a426c77e33f0d86e18bb42a077a1037258ee" + } + ] +} diff --git a/output/pdf/Qarinah-Technical-White-Paper-v1.3.pdf b/output/pdf/Qarinah-Technical-White-Paper-v1.3.pdf new file mode 100644 index 0000000000000000000000000000000000000000..d07e6a2f0f5a0e731d62e04b38f63cf9a267e5c0 GIT binary patch literal 260326 zcmdSC2|ShC_diaQNE$RKDiJB+%tN6lq%zM@G92?fMj|Sdibw-O36;`FrBEnTicFaz z2~k8!6#nZu=R|eS{oZrVuls*}?tR^Rk7r|h@3q%{@3q!mYd;d&8&qY`vN%48m_reX zqPi3}R}Zf(7M7xF6c>tzg%`zI6n;X~+SSV2nd0IlD&^`%aq)EZ_OPO?;6sa|L|rW% z_~hmJfeInAIaL5x5`9@zf_g^2Dd8$VYLpM!G}# z`O&Sig(pST75=c?Mv9XU#mnBxLR3!0#md#%-o;i_&dAIc@90u!((x3fN(F&VAFZ~BtCb$b3p$fSL@K9G z*$Kl^ceb#lDAPaJ(?8Wkw*g2|IbByTm;&fnPT#}Ah1w<3&L{d~INI>bKMe;5Kl)=h z%B~*P6c0Ld6asa8fKn?&227?hL5QwsP6p08e zBw0KOLlh+t=`HZn<1*f2@8b9alah;z>(4SqruUD4BBEqT1Ogfdpz!b&fI_2ivS=)d z0HE-)LlW|1T`Tk97YzVm@JCM z6J=3o984%4Ba1>{G7Ad%1Ahe6gic{lvS<_${gb4y^cH48A&c#gfSQmLnk0)O04OvW zD~rPtLD4ZJSu6sRSx{(%B>o60L{tPUC^`xl=T}Lgv7qRzfclxFndJnAN<|{dl887I zs1OP;kN^V)Lk8^vrYE2=EQ;zEj%Sjo=nVVU^@oTGPe#!MGfq@^GMOekrswbH-GEt9 zF$6U(Dm)1=AfiG87&IP@1)Ly8mL`xLQfG+jhgRWAk0l~>EDhxZk&JfkOT4AtYRsA9=EV3=L0*W18X9#LStuTOr1Q^h;7h(v&sjw}x zCMh;(ok%yf|Ay8P4gXbAXvF?xMN(|gI%7~1l0pFns-%!E*H zR0JGsHmrbRgVq^?nouh|U?3r)LX%}lSny|X1hCs#K}GfQS=?+GqZ$_#5*PSYRCp9k zc+AcpmJM2G3~EADIKV&^6?mw4G?^fZgMFFIlBn3>^}k(#h`El#5P#Py(0P`$iW*`1 zBl9;QDhyztii(Vo6_zB515b=Ktzw7Q5%DsJEWqwd!UG9_clDcEVbCn96+6Vv5Y>dN z(13v|EARrbXaX8=l37$Mc8HxJDm-Ao5zr)2EI3?v6z->Z6&5@%Y75g#kR4)Yh-#eJ zv53?NfC`NRe-KL|h+@GFBqBJOM#T=XGekwT>JYucg2zSpRaW3}K}?MWRO~Q2Lsa8> zg(U(85@5i9ABZQSK~}Ievm`4V8wNX(A*u;kfy+ge%(%gT1$%}nJ!WT+8uezCuNb15 zkQE*m?|OwLv#3|>a63a(6M6-7O4BRYlrg{)a3%sSAxnD2 z4!1KzHNoxRb&*h@RAg@LVyCA|X6gY_MUA*u;kAxn=cD==xW2u)$Y!Dnd=;{QCiW5C$^O|Re-jw(H- z_3F=aI|c_BsImf6nTSS%Ucq8yO|RJDc0|5^zYk)-&-zWTz*}Wauh`*shNvd=3JVx$ zvI2MnSmhWvP-RW8*x`1DsK)m}40v8}2KUqXDF*yOY75hP#SXVKL^Zw-VjxmWCIBil zh>uK$@G%An%^*0LMMYqT+Zm!7mlYKZ3WLP5>G=z2@Vv-K3)86B;dX|o#$|19e z0-!?F9{vMaQFk0@VHy=O8?1U>3{j2mgNP%K{Z&@L?IfgyX;kcRJ400Ce1!&XCy@XH zsMz6lhNvcF1>8=R%!FP6w^OCZw5-_Sc7~`XWCh#~kp++yOeG4j zMbQwxC9@=vkga0Wm3Sm(Y9<(sK28rx&JHr_qmlar`pjLnZsETG*LEIhGiWOAsa63a( z<2C~X_E6LmxpBS%2a(pow5-_Sc7~|N_dx<2!x50gJq(;V6R-qPY8DSGnS4a#5ByLo zYL=!aqm zS~n1rr#{g-qNY;l!d6ag1fLf99%j5Fuz#oxw2m;ejv(ih+DPk&o@Yx>(WM@j=|c7` z^+)JA$AAbVM+90&^pkn|@gO~DOI^j(jv%UsbVQ_e1e!gb z9}GcB>bKXEKG4x-9;FM~37fSv(okZh{RU*%*bi zFwF+YwES3j|4*pKnUe@9&|n8cBn<)`NHCiS$0h`pKp{%PDkz%1{v`#PZoCkY+~QwF z1;-{-;V~;J5<4y+LsS!@f-`smA}UDRgF`}C!$dqJW3nPE60#otjI4e*TSPGHh-4%k z=7(cSgfFNqOv?(PhChO8e3K!9S%=00D#%p@Z=MW#1^IJi1Siv|5T^PgsK$*KA|ySN zew7uZs$p3|g^=AJK{cUQVAkP?h^(L@f`kNzh+w?4CM$${{|KsaSrNdlqe^B%uaILC zmSlzODSrgjxU2|B(1j{13~~wwg&IUSHo>ukiVbR~vdQ05I~;NSrdOcxEb0{-)Xo^y zgsk8Yo~BoDj*3R3NTLL=-dWQtHn^QJstLUUj`&rt2w)Ynq*r7%xScVo30Z-yPSY#! z{D??t5UfR()C&B5mX)0`s0mTw!7u%Vt_ToWV~MWFY*0I6R1>0tGkBU>L0A?C`BzXz z1%4?jOAxk$KP9Ul+aLj)89W)Oi}x4H6(6Lq@7KFhEx% zB$iEpBs$3RWj<+~%m%eHMm3>VaM}%2@UyHi5H)5A6&uvf7}bPc!D%;~wjrS_Na+Jf zfyh{s6g$LDbBF%U5Jf7KsB)RmD&)+BB}uVE>RQ&X9#OtSa|q~rdALuCPTFgwfvSP#X_Ul;dX|w zCZq)?BEM=D6veWpS#Zq528}X=H6bk||BI?x@C6o!fy8zspPd!eLZgsB!J^qmqQww? zUxz=~9S^P-i466*@~xk$;sIq<-L7gT)TPGlVsvTX1}X2Xh?`l>iM64J#4E#-g;?;dq9y zCRQSxc~fOGpD1|7CZILB!;jiqyM{m2 z7;1k?T0e9PN3HDoMOxs6VOWEOtg1gEt#RFgYJNB|L8L{303ZhD0|$Xy7Nvzu$RB|< z&RaO}!>F>E&@J%8h^$GA9hPUPTNBcP*ewZ>7Lvz9ttp0j3OLAQNw?Txd1NSmAELoQ zHZ)Z>6Vig>B^Gsy9hPSZYeHJ!tx|Oh10EO(IoriSRWXaY#SY6egf*dCP`~t>Zb6m< zYr4fwwKEj7Fqp&%X+gO&Rkx_QS441paNzc{rdtqaWql|1B*n zl0ir1EhKvl4l^NVlvUMYhvXTenh+LJvV?HwIBj8JGFenDWKsQ*IUH9lEbN5zEfH)| z6xae-@cvn&E$VSRtFRCT{`-YUEn50jTF8A)EbToQc1WHftO;pRubDy&2(To8x}drc zNiSy&7CR)*5Z1VHibVn`*&HP!p1ZV_!5PDc~sZpP{%BNpoij6g$k$ z5Y&WDAq7jnND74{ma`%$c6gm3sBuYQAZ-rODGYegc*tRbYs*+vDP%iik<_Wjd4I1` z)M}-l1%-i~nl+U|7SbO9H6bWC@ujI0?8X={t&lo7)>I0ai9Z5rT&1YBO20@7+Ytz$5dEQ$d~5bnrhzIF);LRh9N2mt+`(n3RCI|(jg zolq?_T#QR?VP0CSQ~Te*8kZK-_RC^$01Gj!K*taZ3)yg>g?U)4lRJi<8~uCW{l^g) zQWXvo`^ghT%9p4uOv7TM=J{`6O-Kt#%)|jK401XF&J!BG1=qPy?=)sElFUZU^WUHv zUx-K<01l1}#z_l}+#m=I%!`V3gZ*znO^6EYa}p$gW8eq{gGRUuxj>L5P^=s5KtTWZ zB1H0?NU(8ChziNOKw6lW73(JZ-@uxX6=JgE00bKJA8Ie*6LO4LL&XNa|AVmLkOW1- zLgMd)upku!npjaRHu#+}tZ`vM^$Um!V4;y*V7O0E6mmB)RG`eyAshToFT(gc#zKxw zNEj$37?&2D=FnQ0hs6fJGln%TEwJ>6I0P28$dL%O9uOmfiK+k%jW`4sHLM!lrEM)jV77lzMnhGC?W>70CK;Nkiw2oj0LZ0Y$Hr-yM zCiX!its`pc8a&ZDf}7ds9g%4rLFkFv1Fa)kmJL#jL$k4Hc_hd)tutDt1oBMl4z8p8 z(I4ElP5lx{CPjKg)3`tI1+WED zpTkmaE!c@{z@n+%-!VB6GIvnFr1lZPlc2URO_|ifGuE6y2&_=m_%~P+(t^W3xF`vX z1V|i!8)2as0HVjNC96@Z;Qk1#3EhI@KscrO$x1-#IjAknOAA@se+1Tqv>?7nq$aC@ zI|HvKf$}9#GS(7`s8xbLV2$s4v;_(jO)oX1sUY%9Cqa6tA(~!m_$w{Ze&)YOOURb+ zLl{5tERFI1hX%ytOJ86WF&+?YpZPBi2;mO~Gep}k|APbimFhl=ymIBiQVr(hU?}Yd-s@%_(f)e!&b$X(52D<9w?{2g5vUhc%HA&fd zdAWJ6mXou!_pE?!jh@D@W)xqmipM`p*|?%1@A@Ib=_}k`K+Wug+6yFEn3dPEAo2YP5@UK4 znm*`2ih?Ai#MVoPn?o9Ue`5~(XH9Bq zH2KHu{bUaOXH9BqT%5sx{<9`EHN5}V0ny{W|3i%ZD_v7p6!qn|zg?CDd38S##)Qhz z1K9t?0Wl67&?E8x!2$hpWyVi4^q)1UsmEXp*3f@8`1Z?99ZcH`46Gh^3lDo23p+VW zCs#|bukBsrEGaHlc5*iMJH5O;D4uejt~Or2@V03gFAGmc8BZ&U3;c$wr>wJs=RX)o zP&SCXy8#N|pkjlJgXMrhPMQgD^BE4gy#WI)Ok3i=TmtfM3>e1h?d0|Qc&(7PbMN$$ zfudSjl&piNs|&M(#zHYC3I%o@7LvNiP#yv=-$6?7poRHC|7>min}bG%Y474~fw*BZ zKaZKf+L)ns4_-2hj2W^*kc*eF$i)D7XkmWLKgTEj%`rQB&Ka`o`W8p;CZ`Ri`q-+CkK?DI_@&Iq?hVybb5Q9WEXkmWDKU+Qj=7@pFC>9>z z#(Ge^JnWHi%UDxk1}(hg3GVhF!5yL)$a!Sx-hX%v z^-m_>|1X30^tQ3Fx3Y)5%xAtJGQ@wFDoA96rA!aZFBk3n*#Uc6dDy#wkwRxPip($88B0bPl245%g42c64ZzhG z@J>_6V1_p0c`^#AAyE9881aq25 zfb%!x4g|+^|xdFm48|A6p@=!$lx_i&@a?GL5%>kFt5MA7+T|}AM_Mh zG(86vsx-hPfvh8X7A%k-{h1cZMboojX{HO@7Ej&7kxyElC4{$VM2|Gl;v(SeQoDdR zeA7Q^UBaG1eTEUNN8CvlIF7;6(=q>tf)tq_C3fM~P! zUmOry7MH6V#l-_yisCElLh+KbarN+aUTbfyV5qN3^3&AOAQ1i`^nsMoKsGoO;X_Iz z4*r4LX@I~GibhJPiBKlY{3@oVg#VZnri~~&Z(Cb?7h4+(;8`nIIcp0DT2a7Wa)Th3 z6~zmJ8!k?6a(YH8G-5X2V(&%yr%}Oy7z%E$01k#KXB1SyA;r5yxKD}tW<-|i;xNBl zgT?fye0_aoJ$)(Ey@Yzi=LE22?Yx|w{sB7{H0;QwS&#}2Wkul4u!x;4 z8_3$#Ue?vaRt{7G9B7i9rn;WKte!TYLCHV_P}alM8zN7(GFJ9pe*Z8UxH$xNeq<() z;!=2pF11tyHV24Az}1Q*c-=M&`o)%&^9M&W9vww4`=yQsDO!Sycfh|!-bv5eXxK7n z+&o;ZC=_cDzo)Fdm6M08m6e>Eg&W1g9ElLYszJV%gKZ8E;b13dRI5`{=6;!qT86k0|b`2`u+Kj8L79LW*yY*PM~TUT0w%e~A_Su;1Vnd^DqqUMobS zQZT;gwL%zrtq_J@D}?0tBlaD*?+lDDTHF87*3PW)MJ-hMq2oVUlmFGzZFq$!OwW%2 zF})))b&y5PyHe6!zfoOZWxc+Eu8OR)^*^u?2DuamO2de-U7_G+V~F#@Tic1?BM`yu zgb+6ij@K`DwEYVsQ&!PjzfnU;ce5Vz<01gJ!igN*Fb`L^K!OtTk_Xs1krt+D7Jl|$ zoiRO?^^h8{HWNnZkX4@jaqv8GU}T8Dr&k) z`s!MmtV|k2pdsdrTtozscoeJ{0_+a(8U^NW{{c(u-<>pMWV%|~D!NL_>RVWwJmhW; zD5wRm0jwUxSBBt@EDC91e)8Bdf0&;<4RtnV4YE!Nc(7j(cLCn|gokVQ!M;OUm>(Ci zhy9D>xJAhTIE49yhDXk{agdONf!b`U8v$?G0#^@Om>&~c<_S}CrK_T}L4yUB$3tWo zaZ{04DHaR$*myXtCb8nBQcDiUCurPOp>Aqmd;ECQAeAzaXL^8$UW5!0OqgG!oo=hp zE095!6I(m|y?p;cTjiJg{#5OqyeJ-!hAgM*WZ^~GK(TVQrXX6gh2moCWe0E2!9!j* zpQo1x#lo3yXM%HenM=L)%+S{gMz1LuvMoDDxAHDcRTrlm&35qMDmpo5h0v2Im7cs$ zeJZ8)2Fp&?iU=#rjyPrh%B+e1t{!%byeQ}HXqA(1p-oq0SjUC#_^?rT;?qqHPCiZV zK6ibu`%wGFDRD+~u87visLjj8Hcu$nvDmQU%7GUFw@LYm+qT^OT=`+7xbA!L&3EZd zAHOE0-0g4==o%bxbC`Fd`eNNc!}q?1&tG@v_Qnr259TMm%0INbGV#hZ_a=+VCB@#g zTHUX&CCSW>Ft~B3yZX`+!dOD=7i^N>OP~DO%I@5VfXml6``nc}=#tT!>TtRK;+Kml zUC#}IE)1XTx?*5%xZB|Kt&3Nh@9K~0Ud;TS+JEA)`O2UR48OQFaH}J2tr}s3eEr?? zqZ&aGW9L(UdOYGZ>}g$@BpyF1Qa8|1GX9fjQ@)W&`Q>sSr?L7WJ;Im&;>t&sKZHl7 z2@Ng&*YkhhDlwC=_`CI{df}R&oF%E z(%D?owo|BY&Z{7fjhMI+iiY>$0+1u}3AX+xea8~vol!ruA9i%OhlBLX0yvk; z8!+eX5NR~8S7gnaU1hI~uAeHC6pofy)+Wlk0b6or&r!Gi+~M4JZ2O%Z4-4LAX?>kNRekx)>s~TuJ|iL0+jTp{+D!JA->5%3M;}p)%1&eZD;^q&4P@MXJ$&G; zXQt^=P2Ye6L*JLKAN8O2?8XFXM7g2Gmm)|>@uuocJe(T7>f(!b5OBH2BQnl7^e`&R$k7veuvEvJ>9&8#jSLALF z3ELh~vTs>uNzL$|t1=`N%BQ{b zV2T)S>fDss$JXuDS+OwWu{L?(!Ijz)7nQ5+gf1#&ofnv{Ca$fJ!?7)V;f|-`6)JXf z79M_-e67>ci+5qzqZHmw-Kuce^@2Krar-1wR@kkK3yrJrstVa!!H*LxQ+Dq3zzJtn za!ys#o|zr>aT#B~d#CuO-IE&92TqzL3?JK`t~Lp~-@j5=`EJGbdqN*no~OL|%HQ~T z)#^k3)E7mBynBX z9rN^I&NoYO^YfPLnNM|1ALV`-zrROmFl9#Hwsx*PsqawF!|sHYuN>g0ee|ZPpo&<& zS1M=T@O`dK%~q>roR^=gCADyDFFqV%xOlU7s^+@OBKNDs1~^uSWb{2;w({vFe41u~ z)A6(2$<^N~0=;JD<#u*-ZgZOZ?A&6l)4?W-`LWj)f9x|2IJEhT>qdh12Cm}=Ev6Jd zxE&b0UHQrzp}=4>3FG~IN`f_Mqa0qCIW`Wf{b%>cA0AV23F-CkQ3-VF5=&0k=+r#u zJRm(_D^ft&Ddr;iux({O>K31rE2?qLoU-*1oXvwu-Ah`y6*pdL!}p*^?MF?~#37x> z0@&GVYgK2QnSXV<>c!UTuQuN$N0(p@K9yW68T7iReeh#Gu`F1}{K)Z?hKkBg^OyS_ z`Z%I-;Q(N}T6YQJ86lhZ$FhQE-ik5F#ttKi!Zs( zp27Qi+lzx9B)yjFTibVw+xT$!vMpzqEy+V)KWMhOM{-7VZOgaSL()UUj*k92+(u>9 zo|hX9$}?md1->qB8GRb|wf5_rmOIKXSICRKpO-lA$TZsx8SiX()$_@N#zE?1JnF|9 z;~GyjCcRf5S@1R1!$IJqox(t%K){Ln0lnM@cg@QUzP#t|OxXwXTQ_rJZ}iQ+H`|=& zWMlY}`&w7?*6vN>T9vzhY|8HJqarmOOV7obZICP&^uQ!tTDGQS+qtlN>UpoXB<%Ne z+n%}a?n%ct?VgUeyYHMfn?ESrv%5#G(WsI9HErjq=Wjj@k&;^E+HF4>eVXwp>0{E= z8~cp*N2-{Z$-2zj^)Np^^c2LyPf4Dt-_oGLqY)HUAG|u~fI#y4I^}6WMb0gc+<3|t zor|6k-5lLKN#VKbr`xKNj&1UJbGc2maXsI@?xI=N+5sK$ zeC}~j=3L^cGT7?(UD=E0Vo`Vb^3L@8a~~GOS{~?pC>QfEV7dF7 z#T&jVpmJ85?IAzdOK2tZaQ3;jE%L(qA3jmR)A&4j@XAr!+p437Pz&TH3EZCLtSYT1 zT>e>k+SsmpGu>}5m>V)FNBHuam0zVqYWM1&nvD4%>NYqlic9>CsuVBBTQfc{^z>y# zTpx31v~kavb5m$%!0L_Hn$IqFU-?dD+T=A;E^zsGRkvB=#6RnK1vxCu!FbPKIoa>3 zM3%tT#>0xL4HYi;R`_vk<@A5EsO7`v{ny&?7ETwBc}4q{%(?E_@_kbFGBIWB8*&+c`|kkQSoEv|_K?^cR< zR_HXj^Gne8GPBciE?&2^^Q((`@!&votCC7zL5Ojnepy6Y`@Z!L_B3~VzP(NJp~Wo| znd>H)1CyT5Iy5Enz+q>jyR9F#1<3o9N~8|R6L|RAt>@nKy`YWr*>PGn>$HI$Pfb;F zdq{6p)j-_VG)coMe|_`S#)|~??rI)6av^u;qjU52xGq`bXmEE}sIjDF5q~zP$l-$N zT_^9Kt#Fv{VpQyPIM+s}d-plUVHwyOr24oS;xuyma&OGJ}x?UeKZ_O^A@Ilt1-0W>XF!Q)v%_xAmtY4?B!c@ zUfWyiIc}c+iaS#O-Nw%I1AS$YhtEiS*jy#+`Tp$7v(F?xpzvp&$>N5FJ4dxQu1Of4 zy8qGn(|V?s6`7%Km%N{KChT5ZMIIXNNlu%>vwg*)ds^;Y;Z3%_=QuNDgij{jc_eCh zT3Jxi78PM1Y58W4Xin9gl>_G_rdVA}y*re3-qS@!f3KEw=-r(52ZvVmU?LrQ^+PK? z{DV$tpD(I<79^Iu$GF*?=lE0g6XE+Gubnk{#cAt}(=&zLjKnGobd}YMaD44^hIcMq zb47MhfQH3CuVhyw4tYu`4xUsgynD5GtfN0*WO(0-YrHWw zH_uzGZm7I{8s&ZeRA4T_Q)asN9knbsi|EjeC#{TvrEDoHD`Vz4&NrVLBqrYFmn+G8 zK00=Xx6_Fifd}$jV^b=|`qN%jE%2YeCLv#@sVwvLf#44DDG4(=6SHFW3hecVj#X{b zKi0;tHr#xx#_hxuDPa|9j*)jN?9QZ)u*Y@DOTL_wKGbKEfTN^I?8gkMtf#nJ@$GxtETW>u%ksrSBgixno^f zpxCwH*M$p79`9E^=X*KSxc!jvDeGevxLYSWDF;WBNxPz^guZRuzCHi!f-FC&n1Sm8 zxnGjKoOv^)1|HAiUdVGTV%T;`!WQR>wuR2O3Cee)a5L{um8qX0p_be;^-zvn(R9+i z8TAQI5AJd=&YQLME6UN{BgclP>KtX!DNHB%^6~1K@>)y2-uxtfl8}-rEV5q2X4iT5 zStHu&(@vMPFO@tgby`d7rm=d*z7^|&2hN%aAHN}+G4M!Mq&sfWUD-`)wbdeb75VmG zlbw4GuU{V~`^^5sd;QnKT$i3YyHpGg%wC+pC6ndjZDvt5y@Nv|VQp5y`0j-Q1Oua(Ee*q zHyzb)JDNC(=}`n0c`pC>#qDuRVO`n$%FBzYo;}>Q$g^YJe5K|?@xpsU`m~d?&CXaw zt?bftAeWuyx}~!!J*mo@V!19xR1g&|Wfv}xo{^&N>}t@U6xyP`=l#`siTgU+INJ4` z1Y(6XhIhuT`&wwB*)iw#{0jSNzNgkD%ACZmRo(A6Kh<~9x0{jj0|FZM@04FZdnQyq z)9aesD*e#*H`^=&L=^5fAKc*dEWn{oCONbzps8(759WP&O@WnVh2eQ+z6G+J?)UAC z%9|QB9f;Xk%Q*y}X&R+(t>xu!PAiXk(b(=2<#_Sz5raDY=e&grL@h)V9QTI~9BfzK zI;+Pvs4w&Ff!)q&1)n6|_g>t$-Nf=TM*M6>>3K`bAdX90yk-T-n4Q~O9Ak^hW>iNn9yDEt>eyD9OkKmT{3YS>qBZHfou=pdi;Y(a(zVx*y$k)AC*p$_Fyfs}d6JRfcU${O zZOYd3#npmmEf1N;Ql5l%rVLJ=%`0SLx;y1XdO`^Jt>($0p{l841N&CUG);;`8Fs}E z*BEw6-L{d`I5t%|I5f1p^dP45dc3FMYI0`Ii*M>>VFr=8p(osJrkprbym97+C-ZI8TS*~XbaUUM zd`})&aO_@t7Wv}SB|DB>S#ErToZ(iU(_hN*!9V2L!52w(8MZO&*EE>VSo7Yiy1K&I zj;y(?Aa3p_$pyS(!6p`5+L~XxUveb(nn^VDo+p_MkCw;0H6wmWBFB8+(%kL&T4Q&? zEk4&|)Z@b1s|%ViQnv^vPWpMup6VgBDIWV2{Vg@&il@q!dnHdj7hBKUuzXUIrjkvB zI`0$1nAum!TMjn&%sw4>H~!Pt4~ngy8}=3nU)>*0T-D*0cs1dP-=ysDPX3qu8*Gh_ z7rs`u{=%`Oep0YP+ndae$+*v*gHb>Bp-^!f?;`pZ2g8%lF!DttGQ zW66oBU#kY~`MYEp_`1hnVh|T{cM3f2g)GO_fpP_MVF7{NRx96 z{H@HYgx^~BNOc^^Ntu@3>t`R(?|R#5_}ROiMFYFidv|+J`XUj8-EHgH|L~pBo7aNT z`FySEJr6ebKG+qb*Xg6xpSUZ!?CS+Jm-~;!rA-cO|y^RCFJD154A8oyNKe;uNV3yO_5}MA>zD)QgzF#Dz*9@ zc*Cp>7iIcP9QDoB{i;Il)hvu!SYj$DBNy~=>ziv^9Uot=Soq3(>6hr@83bPA&O)+U z$s4IHUwpoOdbM$&)j{LS&fSXw^VQZ#$j&tqS|kKU6vxP1BYc`4b@yFurMW=5%sp}o#O6npY^;1sa<6V{*}tP{^DT6sqvQKEcdthtzjw7?K`a{QgzSxPhD}b zZ)kU*?LDc*S28ayFyt!OKksB~pIG|yB}=zQ+Vy*!?fm@pdyhro#f(-J(jCh4>HVCp zpHnwyZmn$0_O=;}4zt-NNILt}wQlv!uIL=Oz&z`|&(mCuuYau}`{q@=NeB!u9X*}W zAk#Lfx1>M)_@cYnrPYn^?e4X{OOTM>HZu0icfRZ$$NCD_r#ugp9@k`>bn|`Noy}up z?(V$6AhZ9LCN@feQMM}QXV+p@pYY}O-B%*EVvfLIG!ITqw3C-RB=~CWEdJ>l zw-d!9_h=?WUTT|FRSrBvMxuk<&#`IGB#}*vJD-i2WNmz& z>(;YJvP}Pl**5&{DT3H_%N`pY(j6ZDhVhu(a_Hl$f}(?(VlnO%_hm=JR#u18zwLL^2~^2^;hI;wnPfdJ~ox*Z$ONQ7Tbkd0rs<4dgG+>dJ2YSmXC^DTL5yK3!` zPw(~_8kP^Wxk)@4dtIM3qxZeMo870h-M3xzyAMUE7Cw60?A`PH>zlCN{nJca3f#7m z(!8gveRF2*NY{|KTz+^^R(tWSE89QaUzCw=A`>?@A{S6x*jq78?zNy};63N*x$8d9 z3EQf>WU314mY*GJVd$yq$(X7E7ur%lVhITBrqBUkeJ9IAtChFKs(r+)CQE?Wk(AiiPgti%-P++=4f0 z+uNV>3SX6S_4$czgWZY-3SuYxt?%HrUuwD7TH^L>_NxTpyZ4%M3v!>7#ooaa-N#n= z?&9F_FXfk6zSF`wHZH=`(J?}<=MY~Jp+zvz(C|E{Vok3iz!_o=v$ z7pl0@>-{Uz4OYDo2Q&5_w!JvDtm3-@2^} zeM4?BarwN$!=-zDWp2M|SKl@G?DVeYoNu@Ch?MF@^PRs_dPAF=PB+$Gw<*8o9dGNh zGS zce|KwcSP$#|1-4@=sth~JYS-hcX7;5k1&(DJ-^Y>)eN-RI>ldLs)TiTNZ zF&@=}PnNCNB$xSKHS(nd$w9MH{L7PB>252F-Pd|5-aM?3IVQalb$_b;`Scs}dQ=4j zRd#a8PcJ?Y^SFLf1-JQ?kiM|N@7eB;JcTpmazv-_CngMtj4o`*-+61Sq1JSb@8S_Z zjNFCmbJiOsc|CcyI-FlO+$6O0947g?Y2dy>xA#hsZYLUc>Dl_v*}sLWrF(JMUe0uX zC&HBP2A(CDQ|(tff=_+M8s#+WUk|v_`_h6i+;e29t--AG5ob4*3vS;(pr;ZWq*PHq z{dt*PfDD;8Xy4fP#@DA$V*|&W$L^Xh%ipr;X7fVgiSM(97p;@K5g2)EM}){Di&H1c z4DYUYkJ~U!k~7WvGCo}Q{b&I~#>2vR67Q30>c-pFO@FbJZZRL-6rJ7wyw~rc@lW2ocV>K zH6riE7Bkgp4h*6x7dq>s`;#DAa8YgO%KK$r8M9wLUw@Q%slBB%?cH2& z(RG@c_heoPC17Jd+b-*o*j(aKzHmOOk=rP0u84in;qq7^v85K1_4)dxUDA!o)oBT; zl*298&c?3G`*sxk=9#;rgumAuJts7{m~_8hVbWb0l%_!SRUfW>0vX$vJiQU#JlWf) zFS4sOu5@N+pl{lzcbQAbyaLM{XQV1F^Y(_Ao((bqf!E3|Ve@W9WhU$g$E%e0EAJ`JZZ9r%IIA zW)-~J^hA5X)R35~Yj~|}Qfd^2gJn+dKfa;J3*=mun#!0il53cJqja{YRpSZ&Vji{q={dH3wv(5KorpWWwLRyA--#&=g|#t` z4;@!IE;23pwq;H139jB~;zOcLFIoAN>|uRf+oN+$EJEUruKg_WF8u=wr$nh z9C?09tjNq)-wrqosdS0A?Jb)e&y~G(_`%hz?Cj9NI{D-$yN=g61%$8g-d-NNTEhFG zN>+-@o3Z|HG4JI_v+*=<#e|5j^uv+c0jnA^zdu)#~Fw``wlbU(Uvl!%Iy z`&#$D$GWS@8Z-1HcW_a`42j;6h6VSg$_-ie*(CSPm?hRc>2T>w%@vnSpB-xCu;wx? z;jmsbf5ADs3Y#OUYRCJ#OMN`@i^dKW+brYThn@Cv(?@AcVdj<&UW@jDbVbV?{yop} z%f)L&R5N{4Gxx~IY&=krOj6ywG$XlA1Q+PJ06L`XtmIT1C%m2+5#`HQv~Y*~LeGO|FD%?=EHRXA+a|2n zJF>ZM;b`cz+P4axebP5h1)S#@C2;nLo$CxZwgtOpuIc>`^U~|2Hj__-#$cuS*3F8X zdFE1A`dqPD{`hFCFzH#+i>%I_5ZS-IG%!Cdoy4p2>HPFTp8C}5G3O6O8=npC(bCtJ zlsv_=b(?I#;`65#XsU zHlF>uNwXv-Mljk#SyVbzTfjC#Dz~k#=YZz>_YZ@)ZYG8MToM+{kl*wA?W)idSbkT> z<3`__IokBniq%S|oD-W|>tNUsJOvYPrcxuAD6({Ni;zom@ynZsSMTCo8N47UWb5QB zS~GZdh)b?3^V}Anddd(R7AqoQbU>X*3!b?1^5u*#$4tnmhG<|9_o%_K@ z{kf0BFRsu8qcQS?4td7KMR_F#wdEX-^Jj!s4xa5lo1sZQdOle=HOk{b`;EiboU{9^ z5-LyE?z>Z(M|nK^xqNeVcDF*j^XWv>Lamq95!E&w8bXO-D^`>jwd!m+@ukkqJF>d# zEf;R~++`6s?11K;o;ouwslr&51(~Z1TDB^f=5CU3*y9mhZ?VuVW<|H8UX4atwR_sy zR-uQnq+p#Sg$=!ty;9Q$TI)w15K)JHmaa>bNLl2=-}BHnwN(1SS>Dh|qSxBZud`LunpD z8rMEX#)L{qceO+jc2ou`+U<+uw~G==MGH2i7zf_Fv%EJkwL58~^lg>AEL_bcC)2pr^~>J% zVr%o?822Zo+Xt-X#%OaUX{ZvS*RSX$TWjo2Ny$yGlT3>Z-(e?M!SQs#h$TnJ8S@QX zC|leS&Xec7D5V@*LPL+A+j9D*e>DF(6bJm!l&BiYR*f4O4|7U}_jpHsrewIS@V)TC zCDAjuT-op_miPL8Y_RIhy6vxb&M59nm_93Gdcw(+HmQ@5+`TWXi#};j&K3KVYyDuU zQdCysy_I#^-q&4>&@Iz!c9iDcR=siR=6B=R3*uXy>tBl=QBF*A!I~7m@7hS7Xp)8zt z6y$WR)BCP;L%+s-DAijeDc)2o-fpt;zSVKX7fre@Ezh2}^JlY2I~Aj?Zdh6K?O9sDZ$AZfRNX*yuHUg(3(=9~>z?YY zFnViwvHrmqLFL*GpY|TFweLn#KWlXFvM7Bo@N%#vY^h0+CH0uB~wreTO6N zzG>4ud2vqD89(Tz`dK4w8P_Kd`ifsXE^&l@Ll;{tGnv{Ib)6hUWs*+Lg!3- zuf6U0YlpV`gX_xbN4{{aT~^jVWz!AL{!54b14JJP3DmEvxmj_jX`cN?WsU;nx=lBV z&18?69@&%GcG5cKy0Zj{e0hQAv@;^{@o9&4UXgusZ@Zen^Fyyh;tMC&dyZjvT&2n| zIR;%8mwKm?q8wb}LO4r9?=2QQ?4|kb&fqA8IQYJau)85y!8gixc!oyf&JOgd%XdcJ zU-{-9_QJ2>K$85`Aj`dx-$!gq9c?7?hnqJ0_KbzUeYr5W=3ep|PaWRTRhxIUlie04 z9wv({o>SrU(uON#<*GI{c>*~^=ZtchOuOfbS^3Hbil2L>Ck>g6-rVceKVx~U`GO5q zHR%aeGZZzs}iY9d2TQTyW19!n|UW6A~ zR5L;+Ms~{?sk`g$R$GggUqc025jLj<*WgwkxG<{B!^fkoZ82~1TpJzj*ADx2qC$>N zYUO+6(>^mRH+IopCCzFdHU8`fp|hk((VwIomxpxYO!o$5kF6Z@U)yW+VlId9mYenQ zBZ~FMn$^C&@0FO`bl}jm)b1xv2fHg{m+QstxytXiD9y%L>*zwgl-w!1uL-RVzg+W? z^U&lvLby(B*}N}p*o*xy3nuBsTfCMxFz6iEr_m?Yu5du|;?iwZQzPDrr|Yi&Eb+x! zM|jZLcgM%usb%8n+muulZ(U6vtFI-FlKr~Qn67R578q1kv_{tS^H(hw<8O1;KFm9J z@7nhQxl!CLg`q3WzF(U5EpPGO`Fz^l$fRBQA1>6q3hUkV#pGlvIT^dVAZE|E2IGW+ zz=BY}ui~>^r#w_v(IY>&SLa|_=Dtt!$p)JSk7aF9hVf(bHMlP(XxyGcGTUFfH*)^X zbd!^+M-6L;i68E689F$ui0fMCy#4BIFRu9{OrupAdGaIYlkTRJ^{eB=#58daL+_a3 zJ#TY0MDG?1B@1{G$KDu@#^*NlsOLmCf62QwKj?_Lf7HY8=^V}6!LHw>LLZk#bdb}F z4;CCc7+JL59%-D9qi%)N`i>Evb)c8;+Jn;y1w1nv^#&gcdPSv8~eSK6z>}!e8yZ8UiSCGm?l_pEk}n37>6}C5Zg%A$qO;%mq;%5h3y%9`}LPHM>?$n(~Q!|x(vI3sD))v0OIv4Jr*Wa^z?0?9k#bNp$8<^H=`s93#4qZtCw3T-|eaw zTKQQ!NOfjgx?+7&O6WTN_eRwie<_0p9%FXq*B-ZpWbUdryu3mB!P@RAPp4aiEESoz zZSTn)ZgoXcg8Y`pckHO+3Q)vGqfgE{c3Z)uU(}S;A|ZYJbl~XAj?E`r?6+S2_Iy{6Z@n`S_G;nvQN2QEKPz5jA;_Q+&; z?}|sJHqV=`_nBREFD^>*Y5ZR0vb(AHCeBqwoA;EvVvbu$G|%e1a^8ib(yAx*tFcWp zla*qWj$yboRpT`0t#O}{uvhT7(yX`zC+#}k`^<1CI7!}1c(_8du&Sjb-G%gU@W2Ms z$uT)|`QNU~I(B%@#llx(29m8&)dxPQ#uye{l3f1g(b&hT;OE27QXXGla&DWvyw?S- zh_V5VNST)7L&wABJ*o+HIpj30X`bKxTgwN`Iz5~1KTB8d4(_#R-oAu*Z)f7hDJqin zwtC$w!xXvmE6zCo! zAy0v0xTa;V^4=MI(-o!6K0N&xtmiqq<46&QA_x9tdoYg2n*YithmTS#Cry(+K74-7 z#e)}T9WXx7u=UjJHW{C7ypx_`)8TJ0^DXUBx&Ltk5{ome6trXlCjd)MMab5Q>c-_i`NOLttx61|Cv2h*h-uiWz* zX^_4Y+@5Oe|84yx`Ka$nZ6`{#qMM$}KQ|cp+H!#BRxls;up+1w9wt#}lU;hlTPFggFD_*ZkWS(ht6 zg*V(02zCo8QQtt?WOIM=k*6=UaD7u$7qSGiJ4dA9lUN!N|yUpR_)U3j}E zgmUJQ;#Y3vccTl*glXGOsd4MH;R_{e4@v-FlekJ!KCi@fr7@qO}g$x#b}%-=d+(r90MSgCHe$eL3YryZ6TzEAnEOyt-WdEJ8vn%2@^TBe?jz*tp$x7(eV zV3d?}rA94fmX}}KrXi29=t*xqst=srZ29Q1P_}HSu6^!{=eoPitFfG3kKbL{EN8sV z%RQv}f)rt&zKj+o=MQZ99noY7wDdpTAH*KrjG@qqtes{YhX}D}->r|HwDs7z)u5acO zk#y>kxMe4h`c7HDrT(pyl#$_8ipwtTlk$y~+NQc3pUb3F7uP&>z-J0%Z)vu<_u)%h=KNed`}`_1U0p(>j(+>qHECO}83zEl9~71>B9wRE2XLUoyI<;=6a>?UgX)F%*Z)7r>Nhr_4Dnoo3fJA zbmt5i2P$!HQ`CK=&}^_pKz)f87>@E&X}&NIW0yw3UFv*>7R zV^_ODe4=KL!jND8qqU>6#yXRXi;QlHod3RhsH%DG##eFQ{i~|{9Q&pje_Pm`?eNT1 zAWy+2uK4&slYEv>)8ozAPb&;%D;CG2JS!T67eAPO*Cu=9cF;3|+U?E|NvDR&>pf(N}eY*SIpSc`Pe<5%+g0z5 zl12Fxg5kv_+&eSexR*tr_R<|*a4G_2H}q*mxaWeG2?ny(5AJc=?q8;&C>VcpYtYfW zx57h4pznX8V?JJk?-xl!OP<+V!RNe|U)8*wuJHY1cO&5>yD&i%r$t&{nklx-wS0boi7iYN^`Q$+#mbo+bvk<{}N3R`WM&+wLq% zw%>)WS*1~9l{N$Zyma;p&xaQNWgBYZS8pjA5HF5*eHTZXD*S!&9zSKJ)GITL&nj|1 zEow7cqERZG5ia;{%S;1V(_@0m-!Ip6nl&IEp3l4O_zJ-su^5}q?3ag*FMjnvJx%-q z*311)kN2gmbziuYUOZiXPiywBJ7?m&Pc2)@?_hr2^6ZsfD|_OO$3mG!{~rK8K*7H; zj;<}{=pq!3RQ`^%y#I9nj~{&p|NZ{4`8k!sk#EN0=aQqz47~I3wYT2}T6HJriO+zR zoh*v=fxcb=2uv%c*?OKI8*7d&h^;nnG>eNvR|_l5VEG8QSi~ZqnF|z^M0~ks@FqXa zqW4d~pGMaL&1SRP6)7nxE=D=I5tyL7NW_Vx0Nk6*+#ATUt^;5}TiLyGb098SsWUJ3 zrcP^x)Nv49)M*e!N(iDvdnz*06FrnH#i@k@+8r3XGTK#|3**)t+OAcmtyw8CTWMR? z&|v7}_u^Kv5tfRm%=S^!ndq;WFT;Rz7W$hWMzuO1>Un9I<)YgXBW49MdFHnIej0+m zjGZ^?c_96BKIsS>%r9NDOx-G)6Li&Km*2=qhxeOZC1AfCSmttJq$d)Qgmd*M4ass+ z=`s~(G%KTHCovqY&~u9M7#KViy3Pf7CyDQVb@hUUo3>Bycwt-eNBDvbwc{s@z4_tf zdwBWf@li8|PrYYba<6ct_3-@5e_C1eT*tzF?L)cAPTzvYaaR?8yUS#)T{?F1^?HtR zLFyCX8n8ujaT!}jG?y&N1*40FZ5Vbw)n-#jMHRNWU}3Gv?LZrIcc2G^r@1F=hq)fx zOSae1soa0$I_<7pXKpT6A`Wtvia4n?u@ z;a;wyXBls{x&hd~%*yGrrLVr!O(8#1o-iw}0A^Hk0w9s zd`^!QJPdkl2JpTEW=8j=TxuR3CFvaMR3Z)}`3G>)z|}McC*@Jpm~kFU8Jihfm}R!Q zD8Nb>V=*le%F9&@#42(+>!Rq6FEEz!Uo+Of495D}V64awFjo4sZ2&h7EgyBgHjE3K zBvBG1UgE`IAQT{?MFu$_gCXPd_}o5D40HY{cG+PUkaD8fCp)8vt-LOQ4_kMQlpkid zG^hD}exJ+hA;4#a(Ta5Pz6iMMVf^o>XWZ1fX7y!j?>xCFxewRf`NYsMjrT6QY;W>6 z!Vz!orB@_hJMm;Pxo1wr-eE(>eD>4R|0;>-Ys-ED#6y1@$b$OxwXUKNktCA|aXg(g z*&MMT$wcYgqqwR}Q@Qb~tlEeivhik;(ey^1G$Yc?{}&?7Z2k|j%{pRcniJBK&}fLa zw#Kukz9-M96DCMZb_oJTW&@f0_hgrjP;WH7{e&+#_b`_@_eXA{aAa??;b%$PUW%LD zfSXN#8#8)?;U=HqW;>YE26_NB52{495E8QdKjK5P=@X18!>APL7I(@-J9rbX4%SZFEa84*En8S(IxKiki=f$N$J{~I!*m9%D z=Ws`%vd68O;4EJ#jN%*%u)D_zyNEd&MFkw}*h?3{_Uz3Z<3j}i_~?f?t)L)kXGxI* z%P_Xn=y1iMWun>TcH8|-tlPN(D=+j{Z>VgqE_4o%?moY12m$d7 zo_u}R+sQ|Ix^dHck6?UP{MqOg2d~<6{Mu;k7EJEE>CAvG%SN|{=-I72ULpGeZxM`A^E~JjZ{_*4Fo8!m(Md3*Q zPsxmL#**&-Mi|ZWAec4)&NRM^s&3LN&HRG`Y~3hA9Ex6F%Z(1N|xA^DplAa zJSzadgUzxX?Lu9MFGCt?LLZ_tNN}l8xC3#5zWjvFO&~pIe>LXp3uDf{(iGj5V{`Ts z{>|1bbv&2lSKH@N&2avcqi^R?6N!%Ql<^O^gx|^lA zg=RJA@GZtt5R0xB7@*s_v_xY`ZA+W##;RL$H@Vy0yc@?6TQ{fc*v}1m{!?0+OmH1E z(AB53t^6wEBY+#KNBsK|4J>MS0I+Cw@JyG&7#Xe-1DARYs|Lieq;-76v!)eu$1NFj zy!C~fUpR?(1$M6+wfZLR%X7irSC)K0bt?FW!elBl$W(^c2$x?|Cz*#A$<^X8`C@rG zx0!p3GhHLU!@UE>6P5bR=o=(#<2!|?_|GLl=5aOuCQsjt{zx;sqE(zqcVLotTkBl3 zv>VQ)^pU5>Ty{Lx?efv`542H1cxGYY1(G=!ynrtW1TF&S&t#U(k}PmMuL`n95a0|X zlIQ`UB+E!3JOI(}cCH;#7GphDujMcUuCvX5pasA8$7;-|izZXjD|WPz?Ng=2s%#4<@r zy;P4UbOokN=n9{YBT!fU?b=p7$45`jbX%PiwP!Rx0HR`dR!NFosWMYYR)D3`?+~^o z&NCKWyyZ0K1C9p=>v+25hwFf;KRDi>Z(~#c{p98p>i_Bt2-0Hm zR{t~S?jxr^Px3k~@+Is&ugCS+J0_sH1SHY?NiTWwfQe+JDrZa{m$B)%|Jug=>U-XW z_(R^s={DV@Wj~6*8Ro0ba)x&B!l-@^GU^|Db^HB3ghU67LE&_LAeC5jkxl>@u=Wgu zi>~G*yUocADv+fB2y|8WAPQMs^s?Zva%O~yWVYCmWG1pjWKu{;i=b}~G7+R8x{xqE z_OFexpX(X$o(Em^&~WUS^7`w?XdWVwCK6=4_mO8};#*{Uj_rB27ua5+^d6%SVRkW? zUDN=ypN|mAY;Q6Gh!RC{9$PmnV5=&-svK-5fG>>g;K=|w*}6yC8Ut7(M@chsA%!$+ zwzL5*8WYBvMRW~C;%wPj=Jo+o>4Ry@(j__@gthgEI3y1VOZ+v~&DLK7Dy`$J;~ZQu zUuYX_Z{cS0*VwMJZ?Q=hB1m<%VfKk+0yo;ENsYFP?DBo&er~5}r?i`U(j>Zw!)`AZ z2#6vgS#7p*K>|>+PIgSj8YV=N%(4a4q}{I2d1`lcxQOcr*$oP0=zc+!dhyT$R$&(63vKdBP%tfkm^{WV5OHdA5w*Op+f*2 zO?G!XsU;MoFJ-mW2l|=zpm9w&3uT?2YNHffKj0ntkkZHEom*~X;Z8V!w#kGwTYiqL zV0uWYHxWs_$-04V+H_5gV4{$VvZ3I-XDVds6g0M9J<1*`FF zEF4WXKbve3j(q#&otHHIm^=5)SpL;-tND+E#Bq|DC&X-$cn(a7Z!@L@mt{(D zM)?7dQ=%vsCX|pds|B6g!P?bAjOE%?3sx;nmUhbuONT|UNZEdD+Ot)4d)fwF{_p%* z{`)3%#-Giy*b;4tdPXsAE6;w{hGsdYy1FeqBNszAajB0Efl888*deyKb7(pBtAG}I zByDURpzzqiv2~JGp@S85CeTPUaCa~WV1*9S(wP1xoh4RhvU^~2)AO?j-2mq5U@m}O z3VyvWqoQyY-|M7g`^?Pusp#2mx5>B2D|k76=y>vpKYf_ob>P={ z#xIipO1_4xaJU;=ew6&!h<)6W~l(lN8_8 zxa_XhFOn}Ox8fU~d${e=p&OI83P*#B`Llp`cii7-Oo+l0FUJL7+h^M$^Wc$Kd%J}xjW zXH4Fh*o1hq)Z(1xoslymw>fXRIz4}}u)w=Cenb3@oIB$01^yHdS_0S$I%$8n4$*fX z%ENVhz!NADh6_C5e1kaCpm@M1ATi3hLjs{ENGOOz92}7fA|`Wa0NI4H$R?DP;0ndv z0Xn~KgK^!4Gw7xqOlR0_Fs?g8Nbd0@q?UA)kdmlDNKu23q8a9m#z9S3GCXSuFh7*> ztR+}{o@dXqY-8FXcbk?D_4azuYi!r5s1eNB7=Os z0teAJ&Q!qo+hle`7=Js`{9UQPE?JiKqZ4#hf&Th@R!EGpKa>tj^SzS_ftq*`|IaO} z>K=aRv0uHEeD2vUJmzI83$E-xy?go7AOhY={so8sv}o4M`46=vw$$A)^BA7>?mKwi zkr$Fryn7(|;q7H@58}H0SpHG+tt3=V{3MEZm6AR> z(nq-dc6V-EzYg`4^pRfI#sN@hAD=T`ono1lvpi>&`8xacj!p7b$Gx^aj$X&7_D>uN z$aB@{@Hm|gr^9M?g-JBzlSLPO&r=AP%|3r981eribu24~b4-uX`TZ!G&m>&{DA6uO z2GH%mCBm8DNF;85P&D2o7Y*6O0)Igk0u)&`woNTqQP5Gq732rV_tpbv1YY336?o!* zv`}Ky#r}T8J*|a;rvm9bkp9y>Ozwdf4kvYG?5$4yrA{I9mVMTTnhoqFCt1=Qbq-~? z(>0tL5_l!EW$j>(go1TWuvlEM*|nTH1)OCiKM%Hy3DO$gZw}I+ux42n3?l3R+XLq_ z#z;$8J+kI7IQl5*Jn@^gue{MXXxgQzv&W}hIlVMG;m`Qd zO*=2U_pxNTaAe}IuYd5(+`@v(u1>DRLpN@#wV3*^<|=Ehzj)DRX7kQUeZv2*@CGX9 zMzU2bPE?e&RyCdlXWsaL8B@}87&L$o03-8QJzG}x-O#$Dlf$Q&=mafFIP zF1})D9L)#eV=KKrd~2A53w^fINKqj!6nsJ^aVkVg&5?+YV+IsWA++ha4ti)~O9spO z^lw-zEiACxEJ8FVHxiaiBF_;K7Z&70nJ7fUr6G-S(RQ#u`h2LAC4-n_p5^-2+VO*jEkx9X}6PU#dJK3 ziHk!OQS0|~mho-?;0VWE)B~b-xeOWL%B%-+O{JnDI5eFgYHK{j0;B2dyP?Zi3ZA}* zuJpGh=xVscS&AvApZ!!9T|?1IL#8Xwb2-jFVLTr`gu}Qr>?_q5i&=HCMm3dd+8gjHw)K+aq*+8ul|^Ry4$}$pAzn=V1Cj<{U&L z=V1CjX4{nQbynR!1w@8XL&x=EKJr*X+i&`5qcgt9$w%6KD|{V3 z&i5aEg@esDmOeEi6SA6qYgpHyvcWxaMuWVXF!t#+gV z`I&DrFdE<-XXkCc%%~j$oy4Bu1as>1jOLRJxTjh6*lfQ@w|m+s#0BY9EW`lhkP$he z)zWJ`aeTD&1<{VLfI&W3{N(CM)5IMIz}}eazTuVR7h&2DT&-?v~AFRT4asBp_fvB6tW9 z%P4IT|2qA=)I8~uZAmu{4utPW| zoDl?}mu&5}wC~mfi!15xsL(B-=)YVP3ibsWzVmyl>GhDF^TEOW$N^;d_x)xUrU%JG zqus{)SQEMiLG59aX0YGlrN5xhLJ0Kr&F{Zg&V=?s0ZEb()S;e56j$n@!AeX<_W%0# z_{Jf5`K5TN&acho#Y|ozi|}-UHZ&q<0PsA+va&-H22DD$=+lv(^gx=Q877qaA{;LW zL^J{Q$smBtHoL=N^|)Owug@O{fOlQrEuer(k5-qH9<>=>$*dxQ{`NCSVvGWUBt^Ud zkJlS;S~YM#pzKLr^{@W0$#!4R6raj(kOA~5;Bq;gNDBG= zA!Vc)PeLkSY!dRqrXc}OI;hgsf5BiczHOiGLbQc~js2lOe}5?0AGmDH{L!Ze{**9$ zsmqZ5*-vBPZ)0|VeBc4dlKyz(3E0)2Fre%mkhufM+?mA`E}6crr?WsITr0_rrZY}R zhv4n!FLYZqL8HG5g<0}uRU69swa|=K)NEE4$O4!$OuSNh9Wf20>%8LDdByF5gS!&P zFkMfK@x#d*Uiz>gR4Zfu=f9sA%PBqm%jA_uldl$;{GQ~?Ak7=@z2|=mxDWb6$-n;N zwjS=8Z^rU%+tm3NfBRTQrjG;Oa&s-r->@JhD}CWG;GSk@W>vfWJmoL)KGn4l&*zs& zOD&7-^Icc-YbEMDTs zZL!t1RBk`Yt!R>O?KLy!oB;aW@BV)O@B9A0pE=nxYxeBfkG0ocYd_Xr+iD{zUL_#A zL^c-xAm(6@A`;FHLWbjTj-F905Qf%3+n{Yq5@lyO@z5vpI30F7RTR6+?eSQl+BmzF zw|QhM6&21S$sUiY+4tr-yIr;MUYm^v0mJhg=ds&uHexACp()VTR9no(0Vi8~M3wi* z$7Nm~r{jm#>at`!CW{ASN`vy6!YdFcCaWaqP5DsO&yZaduu05-z5 zX{~U#X11uJ!tzq`O>1K>wW5H~W;)}o0!Q#xwxSR$r)=~d2$j2;!f>eE zqum6yvpiI8$qbb<^X{WzwoY|C773SoV5HB3vpanL3Xj|8zrrE`#0vmf*($|MH^k{K zx7yM&uAn3>qrxgPh+~k=;|D*FAN&}^0W`gxCO6zxy6wW@N$7~(!d4PFef5G(8_Y!!J?;@O0O zkNNT{N1>BNMVnQmFz}$(aV{8>WEmO_^z|I$Tp%Xdq{f&$EOXE-4#n(Nt4$?*V?B2t zHjhIwX5rCmOozp+I83tXL!;m6gC#C-*zVAq40h4ki0CxiKFV%?U?ZXmX3yNENvqv6 zqfra=z)ZMjg+aU7P89_%OGo@5Ny15cR4g{hCL9F@%uk)w_6uv<#@IBNgz6&TplQ5& zAjoF8S}#eqy1*HwR*_A7$vsvWCM0MUE&bqtEJ3#}ZvMp2{z5YvY8G5e!@rw2!foZ3 zPcFLQhQU2_|EFUU|HNo>28?k>7;Vxft)G_Er@zqr(`bv^!P=|S?kZhE7h9G{ylk;b zdVe|Qun~uyTGMUNK}i>ZKTcFsPQ0Y^#_0D1s6R>tNfs=!oST+WnlICjVZfzQ0lFrF zoNo=6Qg&g~GX{5oQIUuF!Y8J5v&dQHbl{5e^Yjqg9tu?(BIa3^7<96)vQ=8CHPu0h zNxYf_?QC=l-H+vGfxR(J;fm#Fw^b;oluzTz(-p3A#4j2I-NyKyL(zK>1FdMM1NC_% zE;FN?1~M4KdiPMUoXf;YiXsPu>L46|h)M5gA9lo{Md z4NE@#cOmuDpSJQ(9^}8*dDZv7|EYN+>RjJF+06xTFWg2PFGmGFN;$GC(;a8S|7R?Z=tnm&#a-j9217wVbby#}3p#NyRp_G89!e4mgRYE}ELTa& z-Sn$(y*1fwzWRZW-+JHzyQ^)OmuyNiz2yYlLonLPb z6{%ZX#hn|UZ*bLw23IFnevLk{%9vj;Ma`nZb5ZCVizXP?JuC1z zW~Le$lMxrsNZ(wI8ex}20Ob$?)FEnyXv72pXvXXkSJvzddT?^^t+(h`C%YdI^B#aa z%;84)yP^Cd*=3wuay*s-7>IexbGaEjCvw1kOjN9(%u>9cGgSzrg0e$5EOUk5@IBmb zXt&w&0xc#P^ccOFFs5?$+}Tqu_Z8(u+K8^`^l62S{E{M*TH;jYO)$IqCt(Mo2umjs1{RY}{mr=2mw0})1q zrzX3<@)fM(R^e&-3)7Dvj%^r+m7nVmmU1zttCVRvjKKp_ABd&;f~5_#f#+g;1L1%J z6RJ>-?gn6-YZ|4u^W)sQF)k=?!$%QfMC9;APMg!Kd>N2cOS>Dfn9cf26#VZ>jdtbT(<~ViPrPcVds_ zY}WOxXHdi+2}BDEO3Q`v!dhWTVT+|Ty1~*N-C^5p`>E|?`^QmNSt&Flr6{}9U!36$ ztew#}gPRd9a#T9*bL@3YIZS&U`yC%Sc*paG$UV~Hu<4e<|76$>KO6HhOQa685#kV; z4G~AK-XCxTP}(1AaX7-ff1G=IETB!-uyvh6SzfRr@PIcQX4J2jh1BH9#bMq$qg`ny zRmwCirhg>$5=k}G;azC?{#v=Dy3{5S6P9U?$FcDZ}4C9M5F%)yP6lEV1UlX~ASSfO%*O<}r z#;pu$@yu)y9k`^1(q0VUS{m9c5^=R(bl|!gZj%r%pPNu@XNGIFneS*+0pWnQhEK<5 z^H2jI`u_WD2Jcj~@-#f2(rmWIKi;OTE0o905THC~`kZ82=m_DXQJc*%W1PQ?DF}q~ zWWE?--Wu@x!(PqOq&dFdp%OP9;sS4c(>R+KFT>yL(mZW{|CJn0l>1WH@c~)6Jlp86 z84rO#d`0JToA+wNW(Yrksdb!R@g?TWedo&2zpzr1M4x@+pM z*vZy?Z-;iwbiWjm#<{T_Ur$XoXSQj!sm5fgjEqFMNF*~{5?&DQkL-zv^E?&4iqvJk zWvOkJHv1}PoA27x&6XbfCTFj&H}!brE!*4vw}Zd){Kfy5;P2B;My4V`)l}px^3F0< zI%B3~&IZ#4)7xo(7tSj-CB-3t=naLTqb{d}9agp_HnH#)NSfY>Z;y>-wgj?ISgB%- zS=U*It%A1m&Wdf;8qoDV&l#)GXAISI%}}?MS%t&pe_W1Xq+}g}PM1!EE{zBex424( z+fext?1svZplONWe0)iKqKen8VFp|H9F99q=^na|j?gnyh|o$}PkG8F@mZs#%pg`8 zYhDzoo8rtjbu;RwNZo8Wdyq9N4C-T9rUBfpgqwS5Fnw{^v`K14xC7drAoM#0#6K~; zL1=0WmAH9U>%?vw0D?Y9WyO)AnZ21Y%_%8P=Ta1sm6^wT z{Yh4wF8l7mfdlLJZ;MU-?FTPx;YwG0ZO1eJvVF%hrlS*ozpwtjR|Y3PntbyS`rxq@ zcfb0x6ED9CGOS_h1O7C~ja05(>%01=&KRR1*)Rqm8GzFO6a!#Ylst!A>bS{Cos4}p zkbaPif;((A2f~7tI#SFQhHEq8+Ke`q%nHM`8C(6UufL2-ipooE#mu1vCCNr3;f0=s z{w7b8f1PKY|C`)5`G@U~E03qzEcT$hnd{~^o3`8f?ZfsbY=@-7@*$heXS?0@dyaQx zu66b~Z*uZZ3SuCZI}3M>nfP~K#Q z2x_~YO_B|NjRzyy)@TH}XOz;IBdRSP$8>t8F&%4kj>d&m@IpF^A(1UX(n>|rstu(K?}Ssl!-|#N$~i4y1kb?Aw$7HSpn`-~UZyfAFT&cRc<0t()(qxA~8}M$@SL z4CQX#|7d7S&x^l&^G9DnovlScyYFfC0yq!y${S=(u;jVb*^C-a8}k>H2&&xGY00uJ`CfuE|E1Y7xG&dF6A+* za5)%NxC~u&m!XsHdfvz#u+|~BC2n%ouurSe+2Iy28V(c22)3wHpG>P3ab1KqgFuk-PiBq!KBV56YGe`8)wh#fwtT?4F?lG5O zl}!ISbbwFM&b{M-sW^yi)p(`V?9_NAI{^;|CmFM+S%zWTJCY6H&Fy;WrZ={4e*Knp z4;GD0sLyQQ@h?waf8C?EfBT+KzO$F|cU@h@InFQQ+&_QiUtfOv=Pzk`f=fX@rvtxC z0h;q^qpJuBr*JFyHdC9l!rH}eG4)AZR!fR@vJGPWRIHIfX<@X4>VC^~-g_=pxZFK2 zczL+Wy*#xle6@Q`urb{3-kRDTzD~R@Wh=mU9`FD!W1HpAeE!iBV=exe9lE8sCTllLI@^ssUpTKQS=oBAr@s-47LSkFJ#gNrXwmFtvP-;+&L1;RwibY%4 zW{x-!g^`Ff#ui5M!L5{xh^C9%wkWDqodQ1~sB<_axjb%n4qj-M=PD+)Rp{2%bW;_$ zJ17)OBPgbZK_ZxD>A-Pvg3UGxijPs5y+9v}+uvenaU{P>Z zFc$0&e$)0X`%`vHsy*L65h%b`NaVd<>Yegv)pexB z6B81qCJd(F*DWGgVL%}ep1Yr-8voekb8(Oxi-semdk7827#j-4?7)WfnZkUWCCp@u zi4^K{g@4l(=e)Y&+@G*_h1-Gftc>xg^Jub{d?yfmfgU9plHZl-8L}Z_bsbgGru!k3A zC+fb*`Wa*;#o8{8tl1^Z02uHO5LZlEr7+{)qkH#yQg7L@Y)xoxabxvsukjDxvu#W1 zqLuDPRDV~QypcFHxG zVVmqrDak2eCsrh%Qt`JW(@};=)$A5IClEN{r;0!3U*{k83x!aR+=@b!MShe;{UlF|21nk-_WnB`<$7WuXPUI2jDJfFm;&g*H~TKq#!nnN-p21!N?l8opi>2HZ- zA9KFue9t-M6e7+_XFc@w4A$b*<<-m5DNPY&8+4GMR|L(}#}ncSo3KX@7`J8OIS7Dt zF~yUQ{KKP-PPOhm@G0OTKy-d)s{_ZId73p#CcXT#jcu~YHe5ICw-zrP`tltC2i-C9 z>odK-`05K^dZO#s`+oSRZ#?nk8=n0Bm#%wqRqE9_#ht6mM((B+?>t25Jr50ky7?ck zUH3h{;1|b_{rtt3U(|TYZbJAEP}k_E+I*c4+IWheO<$@rgbTc8v@C_?QKzG%m23M+39p;m|D_?his7qd6`3Wm~$MWEkx}W;9`U= zYTQhs>=2;PVXI7J-UQrh*1;Z=nHs}IV^g|zZiXw%g#;s}xON}B@%WBsmmb`{rQxd; zFfjbv{cVqbYho?;=M6_1tEPFiH< zT!EzSIC$`&@aNZF`y@rk{p4-JO+7YwHJyj_;-l)b% zz#b1{G+N#OYzAQ0JDE8xf+@!|Unnu%ZZi2TCX-nZIKkv0)NbW?uT5~7tmdRT$-+9Z zisrD(xd-||em_V8dyXvcvC@dO(pqoj*^cE{8LI<*2Qy9(S#kQ)nvOGBHdYN*3(jGo zJh28n-tT8DzEF~|IiZR=g((hfBbCe96i^h6!^CUERZ_BBv1lft92UiyYf%9+`>D+Ye4|%A6A^n{%+F>g%<_hrjs6 z$Jc+uQG36z=97IdE$;+&2koB!2eYrhyS06%j`(W0bSaHRY$Wk}?%j9Pc z!rg5`4^)IX6T20BL_shKVwtoU$U(eJUM273x65zyzZcC_3wugYai&Jmwm9yA;Z(w&gF(htn{MFJnnU8v zCrq!#6bvH}cn~m9s-=R*tG;w?Un#e{G+Y2-rU?2bGKu8Nt+gi27goKhk zZo+lSz_kS;SU{u-q9#f;^+Cfr1S+zafdkT~Kb+i5kG(tj=uM`hpT0mxCU;D9a*;1h zUIXpu7LcE1XlLi(Ly@&cI{4|)j+c!8u#iuEG7cBI9G;tMV6l+D#F>y*yc z?z1k}?wOk9omdX^`JASRX|L%$lTZ(rGbTP_>NgFWrc45$MCN#{lf+`e&Qc1DkoFRK zoSXrIOzJWHBhh0@OX@Lc#Ho${Ec)Qj(5auAGPLYB5XjGhC+EtnuK}MY3y* z$Xey;)ppB4G|Xf4bH!X3lV$OJsP}h2@g%V`W!hfa1nf$J(~@u;PJ%;8aB32qmSll9 zoCJrG;M648mZU&#Pl6puurmqvBn{b=B-otwjN0yN)0OVi5$8oXI~DN z17g;Zv&ThU0?uIKUQ-vYp5hFq7mr@R<0ToPxHz20L@DdSbyJtIy*M$OIZoTg0>u`PY#O@}#HWuPJ~d`S&c?hrh|n5sf;VX=l|$A!=Lkr0_!1U0^ROX=tIhh zz{$W=Ku|4Shu7zZ@hcU5cG>Q*Ii?L_9Z4@ZKby2;Mhvx-&z+y{Sh{b4V(#Z{egXfk-+hQ z5a79z6rVwRK7$&4M$gNKJo?;0mrFM$bjiW1e8J$=l{9!2bdG#H=Du(+T<6ClKH`7c zaP}+?Xs5l18g{|Z7Dwgj#E8}>x+K{mn`K^9a$TY$M4hr*Cp%`&a2xABgkq9i_5pG6 zCO49l$(S#RlaszypF~T$AKU)Ux>e%;A)PzNP&Gs%;u1@X*lO7*c3Ui^ z$~^Zx-|Rq*vedoQR})xcS|c?oZSFQ-V_>UktJJA%b#L``26oaENi^B7;hRm(@-?;| zzRT1l_t<2ASTMVQM|!8tzIHkp#9m3hR5E7g@e(10n*L?JaRAYf10o}BQ zcKDNSeglEyF-LYz=`1rPX2q^x3fE@;z#RqH-eHpR}S^=>+ z*eoyFz$Un+OTtK2ID4W?orM|^X}a-`P+$C|NIr2PVGPc+0TFRNbfS!u;7#c2!D`qe$EWuAXt=qmqJ zp;pT^wl)4Wq0N>pwr*vse@p1N`b+OSfp=1WnSRQ9D*dE7rTVgjs1i+?EzDCE30Enr zmG`ZGPMcJ$E(h-mGqnxT7j{^QBRGwuf=MJ5jFVKjC769erivVs*U7`Opdw+ZF@1yH zA2cmrIbdwqGe&7~16Z00hs@fE6F2V-QBMh1l2}eO*pj}k7z?(za|pS#Muwq?p(H6{ zDAN=%&RwX8fy!qfuTT*qvbZcjr)gru6)~dh44Z_nZfgW>Nt91l!5|Ko!-vzf4lA%7 zF+9#+7MEl=t%f?Ex*YSRc=3dxJQtrtGP@t2cmJk4PHf)(-u0{Ro9TLD$93QP?$F?Y z$!^mR?z;Nwd!`=x&g3U|FPk^<3IF)3Fa7L|pS|)cq?;v^-TXd>azI0Qmr!-WVUhSz4HF=ss9iEQRy7b}n*Tpy7e-J;k{VAZ# z;4*E|lyYvit(IG4Tg`QIZ`pnw_`UDL;2%Pta!xAPy{TcV*&%wv0uYPeQ9`EC3z??c zkXQsDY(d&d6=%%3&N=K9(s2JaVLR3s4VbONC+UCm*RK5;*7)w!_X>nnaR`BC35Z~XDL z$0jnKxo+@@{X4dQXR@2K%&(&}sd?YzEl=Egej)$;S6_YcC$GQx6K&jb8xYCMKp!rO zXUWFq7kQ{6&@7=;SSU0J8-yW2lw20cBH2AI$xe6+wIV|xvXsBaLM@r9hkCe7m(eIM zf$z_r{f>|2{t4bZmG5!ShcKBaY>pOe(5Ozux=3v*_W{kJ`KWq0K@*Iv0`!Tf8z=|b+K+m_7x zZr?2L~Yf^5dC3<*C!sSzX}yOulcAkqY!odjnk z!I?>LMiQ*XJCirG2$|l@dD2xT&XZ}#u zQ1)>4Kgr*+-Jk#U!UtwOEP;>?(!M@;oK_L>n zFUSRd0A!Q`6pk7VF)s_QFgt|`JwZ85TbfNdczva)>A(z#QaW=@T2C667EUn>nvyL} zn*YI=82>}e!)Q?m&$LETX=-*b<_VM*v(JlhsXCxt8KVgL7&!-3_GM6IUj=clE{JLe z>}{2#Nl)&Y#OCcWU^xX4-=T2%i2_={l4j8q7*=Qs3^rGw8Q}%sd85+DS_)DzFBy3S zrR$217ju=x!^K=NQ{TuY0d16s%}3R$0VFd6SPmI*gk@6I8Kkd8&32-&bYelAs?K%K zv!NtFXj@HR?$kCz#au1U%=e6G#Ng$+UIlQ(^G`r_`!9i(3^yZsyCQ00zt!cC+Jg7EEC6uOF zb92*XYDdw}v<6dCQ_`>(n37Nsb}{AP3(ld>%gvj~%`Tl&c9H!I_%X9%xFlOS>b&#% z8?Kw3^G`2-qrPfx!PlC;{Dal55!+z*4V!(wqR_3!zTVRP@|RzGi(V1lGSF3hMOGlE zxc26{#k=w&(IwY!3^cB3EXxX~dF1SpsvFj<-n;S{F-ynY4i*r@#w7DVc zG(!Uk*qj85NwCc9Yv;0g^5d~=0EUBFl-o=Kbk2bd7fb{~x9+sQlzNH@?9*(&3KQ6_k`g?b&=@6z z4)8v-{l#0Eb^PL()I*`y;d00l%5Pc31B8oavRU zJ%zX4I(F!gCz_xB=w9W@uE)5Jd#Jf*@~ihu{L}KnR83yr0sW=be z#~BE}8&xmjn-Tmdr~UD{2ftTW!q8XE{W!hlXLApBhV&^vJ?^Lex>U3;!loTk&!lqw zsryn#Qm0ad)cB%Xd^t~FsvBdg`ti<#M2VzIC!~{-AQ_zpDc*U|7xZOZ#>ZWXxO9*6 z4iYY!OLf6%)4ux3_{9?pT1Nsksw<4s|7OIR0Lay*exz?DXpO3 zW^KzYbm)NmT{v~cTYVxDLZzWCsOKVhz%ue{-nYrj*kSPxoVy;om-@BD+` zI=EzO{p>;R{)w@#UcUJ1ru**T%0GD<=^&NqRRA5xT*pOw1Cz{rkV~8S;C%6^{p3!$ zC849uthFTTU;9lKB3nc%#!Yx+qv_V7XtE^_G~{t)wuzEVSGmk&x7}4P`P|`B3%hV2 zu*Tp{^}Ebwe5Le^Qk=a5bfwL*H$Jg#ClgI<+s4Gs#I|kQwr$(V#I|i)|I9h(eb2ey zz5j2m`|Xvzt9EsD)vxO5r&iYPe!64f(6Fz?QrHUm!O)ytIUMyq#^iE)2TaoVZVEJ7 z(#H~$q6ciDEVk9C%JO0Cffg-}6DtWhj-@S=_SsE}+S4u$EDtXkH+5PhT>UwTJfu7W zIY~afvFY%ly~I5!yQDUvKlwdry!a=62gqWx@w(e`>wu*%#3fUv=vr!*K!CR>X#2z_ zzaz#txM1{fOov1Bl?++?G(y^Z$Ex=2Odrv&tCb?X6a-+|(u(NBBo?v}AT_96)*Xq_ zuOb@pKWxSx|K=~27*eik2R~lhk7P<-`%}G;7&T0+50Nxtb}A>?WT`*pTc{X0T3*Tm z&pR05(QHXiNMlIjT=%iO#mjZobpo^Uvwhj1d01KJHt}_mxv~E;YEP?lBbDz1A}4V? zH^=gApFCR(?;Xvg)B|J-Qj_JG*+$QL&Mfh&j0f=1jK#|+6tB8Xs=gfukEGmk&1qTG z%eClWY8+XmX3qdwd{*NVa*@u!k|z~Oqqc+?xprj#gPGBHe-1K#Vg%n2P(ruam6$}X zK0v=8L>N=g_A9`Z9}%;taNE+gV|rr#w50sQVxTg>K!E`AhH}Ahu?zjX4>%g7bi=>> zW7+PqFG&%T5_Z_1^oK;G_I0`33KM68d>L!?g?1&jc z5WXUIUdM)_zt3%4M`AejV3>sM08nD?r?_F-q3Qug>GOc@So>P-TJ zGcW$<@NAU04sU5!g-?lhK36`sQ5SVPE@b9Hbp3}}I|dEx+Y1|np>Z1xn%ip$vzj_Wp%#rE zRO>ygO~L&#a;7^taHKEmt9?w@EjqhO)!~4fds|N~Ul^^I=?qbY+EVJzcSSq5Ti??y z98&5p4-Kj81JT1>+H``Ow#&4`vbILh@NBH0`sN4pWMp1VWo;tB%&u*}ecz=sGoL4I;5{C3*lB(CS5J6&2vUg8em!f?A9GuZJAx+QHL5za$FSU%6%FI0Ui&B%cC~C zee2w6fAgFtqXyxg7^ zS$(Q{MH#ks5!?uzYLxYEqgP$$&Qr?`^Jm~4)N573qd^mgPR8K5E$CUOPDUV~biFj; zXu5NaLn=(}u-Su&AOzJMYX&5P?5_PvLbC&=5Nbxcypk3PjYN7ZQj^SWZN`dStkhgE zWNLBD>H3_eS=V$T8ON&7@nACadLSWP4IlPko<7$mY~){M4kz;THc6idT;arqpiRX!2?3T&7GmD%4GDId1D z*lxJvKa{8PwZ+-S$9^Q~Za=9~R5U9Dkuxn3hAhK-l=}fEaI8mj6%czau#xGhF}mYA z%&3!gU9Vn_L8Bdnznk8e-Ca0sWU@RHB*~ESXmOhHjgZDQhIkbzH%K?qrH=lYN~4(4 zAb-?LZ}U@QsCV9N+Y8^91+s<=jNx?RH|7X$as;X#=o><7rk>%^@53f6WmR3J#y@3T zKHaNuXFakHQ<_Gy<4`mkD?Hi5p$skTjKg$hVk)l2l6KFHhKXS7n;W^}MyH;9Y-D{L zbsxh8!j!u;N+3VeZ4|T2;@X<1J6OIsvop6ltHar}R=r(+&Lr+saw;ogDp*}VKbBr5 zzNo&m^%!|Ah+jQ(YPqo4+-;eCjeg6&2ibR7{CSzR$K(2GnI+5pARREfp6T6(z*2Q^ ztI|i%qk7&xKf>X6&2KzgchUS_PhT`!=8;2MleK048t(gD?3l0O^cYKikt$RFs#^jl4MBbk&iaDLY*l{Kutz zE}2TK;-ZTB)Hod#j?FhTSfb{u+th-s5;*j*A-#}j2)CjQE)6AWqr?ObZz*hXGN!@8M5c)9I6RM0OSST4{FldM;BOuDIQJA@$Ze3`7#f`l z`SG*1Es4}SsH&y~t_Cm`2dPwL1)v2%j@CGd<1_d6!?qpvEFI&vMR#;uqXE8&8ti?D z3BRd6b)Qj~7&>hP#mM$~y~W0iJrdi}ZEuNE>V|D$_p{-8e>yHPFHV@p2itNm$}`Kw zRZE~a46m^|;QHf+h`DFBfrc6zhCw$EW|(XqX7?l5mD@pcOt4BCqkm89u!Tl=78ZX* zQj>WZM&5={OC{wiqL1%E|Dg^vqkxwH0h%7V^(xtjNBcdpf0&XQt{?S11oq5&JHa&W|LUs)%91b8RApAlL^t46O}7uy6-mL}*x$ig;UAd6TQ0 zGIymh(lT1P_xm6|KEJv|86+53CfpE-evHD3WN823Ac4k8nJv!#Iku&R%8oFtwBjPc zq65dbz|=zj;-y!`E?Ryy$u2|VU9;ZO@iveG`8lRQJB3rb&fusvh39ELjBVZy_vaf{ zmlVtTeD~M05NIS-g%!qKMmfx0L$g?V0O)T#zir%}-#Lda@%>dz-4&eg2rMqrE0_gJ zK7Y@(VJMSMG$-IjmG$5E62>W~b=^c^H8`pPKjHDdd5J`MomA<0KV$5!aG`js&}Cp%Sw(K4eaRR&so#;M$%7@1zCXswhjkD~0@f9vgBwfznIeloBY zP>1)^J*|`Lc`5_OJ1=3)fFG(W<0;q?5HJ|62d&oC!&ft6<-pH4o4n^;%o{Jac;=M( zrd4@=s6~+u{-jZ%xGLLgl}m*S<#!9DT>TqmUyn%uj;1pv99cM9&XS{0>=`r7$w(U-h4U;NQMyvU9v-_I7gVpEB!+w4NA$WTDSkcjBx84Rb$RLL@lz`I z<@s;lUydnDFT?~uvUPpFB_Shv>Af|Cov~sU^2u7x2@n z=N0zqm3d9M=Qi(KmCvQ&akc&jIYcc7E5e%*FQ?P#{`l@^!WaT*ktdXaNdRjVpc6;_ z^tVT$twerwxHUBIRVgq2i)@}0SLlttj5WaHOT&+?()(A$;6r9-ZHTWkac-%;74dTm zHgohCc#j@t5_A`iKv~oH-c$kKO%jxCImPocY+0dzN`9KZ#26#7gdC33-Qpbh9c-%f z%4OW4nblM*epK)J>;80IE6Hr*q;WBF>DuuiCg$45*qj6!=$V*PC1AUgT}n#KdT6@? zBZQjcnBy_Pop^aXGqQWJ8@e?@@rQ~OwxZ-@x0@|r+J8i4LFG|+GZf3#mmkq2CN;!M zzfEdUPS|9>+<%5sPH)-KD9F39QiP*EDz_|Nu0P5`i=t%HYV$mvHh`o0R9Xj%(3LNL zzDL2U*I2yS`|U#UNMO)l7bFVe5|Hc@LqQ~LN!q{Hgo1!1e@=4SjMtkmG+xm>=0i(N z&wX7yw6j&~kW4zSQ3DGlrXxwzeLkgw(7RdIXE-zBArVBSk7{#NEqJjLQGW*O(=}=g zGYDnP1p8#Oe;gbzB#W$Ko%adw8k_4)wwID&(OBzm4)4$0ALY_oiYY9 zN;Bx(aAFpmPTt2z+`MEQoB^7wOb3EKHS>iEXTiF-2NRY_ zr7s8c3lSr>X?36g40GK5b{>2AwyYo1mtcKc3W|`Dbbs^r4QfU}HPy&w&z3Q?PQX4@ zE3^(=_r;N^e}~w%JZ&bfzlYcd-)Zc9*>!Drv!kLutvLBAO;I$m84!ZU13 zp&a{UiDULy5xVmTf?k(EU1-9)f@yHj5!vgEAUdnTb!zI9KF#T!bwpRjOwb2=p!Ij) z?EECy;0~@!v;VdRV9nEx6Qe9~fuRLy30vTwI^(Rf59Y=1@%>aVdVI|QxomQl((o90a^3V^z zt-h;8ys92Swxw5sVIdZGHgJ{V-sp0{a%vX-PFLC}w3)&7{OA)99Yu5}sv&piZZu(4 zqweCc1S{IiAbL?NRM}tO3-@I=_MBlj8wr)rVH|Z+RLRxhyRbe)?%ZsKfvJ@AmNMWQ z(SG+AI{>}4-lw)fORR#aTn^IDea(n%_5J1!MqctZf!Rv49CJg`Kw=sbw`U_Gwezv0 z8AK{uKaWoGESIQIV)^A66fLeE6RXhQOP!BhQ5du{XXbC{>9G`u9#O;Tp!2~_r_m+7 z9}v8$C0T@LSK+amGc;4c)1p}=lF8ng4B8JeaPRzL>rk`ppGtc*o$b))QEIF=kvBL$GTdQRFB0b0(S%Mm!ui& z_8ll}zdR02PLFaHkV{Yr4(OYT~`T zH{JbokTQjD1(#PsKM2yR}vB@G1L zS|~#J(KlyIjp21AL{+F3Pw8khTk}_Nt48&;a_4now08ZL#9en>Xjks-)m|?3m{fbs z3M3O5XHD3BLNbDfDn9#M8S?>7Zy!#QkHvrUk!q=?LbRI)!sCe5>K)lV)eJs!)Bp$e zr?EtYH3@`!4BJq1g-*SA1;&eQ4MpoiU=!cQA8W&WubMhB?a;cA7+rgqES~mpJ_d^s zccXKxKrxABQO*67A?uE|k`RUItRM~BG9eZn6}~G0;aRY<@ModA`Z~>Oek%@)AocFi zMMa+CCoGkZHHGY;i49|K18k;x%n;Rga_+jK0{BkY9o@}^XvJh?Wr73dGscK=yB#V|5VHg#B;2>4ILcMFt$ z&9Lj5x4Jtt0u8ju!|z;0iot}a3}JQzd&U^tbekyFR^HZ9$(|^Qs-qS-i8=`Fi@)Xk z*nW{s&W)A^=slTCZ&tJGKC6Rv36_+?gGtb`K)g};y8-Q`4|Jh#Fz5Lf2ho((A*vr25ZJXz(= zf4H+ayO8C0d2xcNwo*WSFGplbTK^3>KlqUOA$1b_uo2lANwwvsxLK&>F!C_Lmne5z zc)`~|%9aV^0d(SfanH4mbIw(o`Q886PjJ4c^Bf3A(|VQm>!=uWu|P{8rwPM zve8zKR#&jLgEBu&P1D7Nd(Dgt%?#71gEB`)7)NOP$2ve=%nVI|^K~vihBDCBRX5Bw zX3I+xJlbkHI4;F?<#Fweb-Jk`xs2%op}+MIP0%7RGp(M;Awy%4sV# z?S3cw`*vZws&ZgWkB^V`0%9z8_psv%>aUGu>Z?vAt2tu=&~0PtI4ANPpsg!Gt&sB* z7d@Wc&N(X^t0fO`_I<0c;4>hP4}1+>X1~lHr{D#fJkfL)x;Nq5y-m?+yYpue8vwv~0@QuXUHXROPjqHVLd9e%UklkO*_|-xBfCKQPwJtgdWE!oUhYK^` z-QV32KpYiQD=ap_@oT-h?&e!4@P8EMRM!M^q3{hi zxDqA<%!ldI4bu@2y2mW`Vy=s8%hl zekCvjkAwV%`Elf1#SB{yWXKLR4+AtM?&HeN+yH0pPtSl2RGBTjZP#GMRX?E>81J}8 z37YZ#DA7Yd$IZF~Min$IVx&FvP$5frjtO*h0Wh{#hhCJRWHF;UN@MgZlD$o3cHv|@ zc6%;kT_7#l39$v`WN*;cs{P^^lzo22&<4hM3`MS6xL;hQ*z)}JY@_=^GdTL=TsFH$ zYUvlj$^EM8E{=XCdjc;+SoW5mW_JYvsL~>PB6W&lMUOzNTHZ8WM9NZ2iPh<0d<>J{ zIZ#J*3UzpP9h?g7fq|X!P&D5!2zJn&%d^2i78%0phkFl>)>tCoE1m^?TAuQld(t05 zUQEp;IN}YRW@@7rGUP|!Bs{IbgF|ezjoZ}(J~@G_9~$$8ElRcL+G|T|gexrMurN>6 zP0P$m=i5t3oF^z0loyz3O0=iT`GW^bTMv{PJY&=p7fY!tD$1x8%#@fI7Z{mJ?juak zZvqV(n6{MFo2-{ZPgYt7P5_@OD{+B;Qq>Cv*SBaHK%D%XUtp{mPF!T9)&PuBgfdN< zXKq3*4KqzURY?!SpCNC^DXnR7+zvHRj4$c;A*(*2gsl}us`!UyR zpx7`9HGfjNuNY=_RpuI8fZ%uBqTZ4Z+JWCe{-nL6p{_j?T2tLw*@A!zl@lK^el7DA z^*nN^Vnl-!G(i~VDS>OC^iwV_q0i-sX}t=1>AVROAm7q$Xkn_LR;gKOBAZko>C4MAjTxRcZiYZ1~ANyb~>8^8zH8=dv!waBy4hqhK) z?6gM>_kGvq%iL#cLk^9Sey2*zv#!v==PL?E1dCS<`EA!>X}FS`cKMil4gKxF?=dtA z1~l7S8YQE`stNR^XWf8p4+)t8r?zKQHm!e~CGN0)Hk;J5T| z`W&p5mb(+Y+N2Og2tNJL<|W_Kq9rdB8hn1MS>xU7hkApyMZ_MOM2{#G%KX~QxPJOM zW{EY6=|GyyQAu;K&aNlyscS;pWj#ZlbL^tFXDBirP6^%gX|IU!X`A@!yOU>VCXvc0%><$Z@Fc_E$SLD zhjU22cyoBpCNpwHb3l=vWxzvlji~oMfuo{hTRMtOW5BJfa>rBT!H)wp^3W5kYty!i zn8k(^V`5tk;wvt_EiIr&1lAZ1h_1)2N9spv8{5n1@74)h3%;vR7wFAV-6<_jTcy^; zdr``u%D%hJo85}ro-hM1Od1($tKT}tJ782__l;FuuI<1D!|d^6^MF30VVgY$cGtYZ zJvo7XkfWJm{y1^rj?)kme;S)kAN6?y%tKj^vAKBEGi9u;GDUr4|B8xFS}Q1Co0Pm6 z)cUmh-VX`D6)Br#gtnP$>%j*L089Z?86^F82jIow)&@~s_dqw3cbzq!6Fdpp(`J9GJ2fgC{PUP3og4R@k$`F;fYiibyK1eNk8VANk{V*|{GFmn8;B4aD{jW%SbWkA{ zk^Q!4fB>auu^g9>-dnZG>uI9{ws>!#oV5}c9gjf$)GbXLK=5sQp2vpXOTBe^km(SYQf^;D$s4g($YhBh{tli5qg*|&Z5}Lnzw%oCyw-O7 zt|+$FvE~u(VCQ!C>XC;BbXo`(alnmNBA%m7dJHIwOGVnk2zFsNg3mokq65{I6ifTa z)r=MusJ_K;Xl$vo_rsyO3RdLnNXAfiblk&lZ}@NSd&(!xyds2F-jk28B3f{p7|$lt z%^>bt!#G9O9HqKbhrM=$n_eJ(8_+bkxDji^mO+Htur!h85W8X2=K~FXSIBn4n0Cq{ ziM;Of+xIa``28-dau}V=aybF1K(1{9=|Vl;R_1rE2(Z z{SFilTyL@MYFeh!bt02z#|Y{ccN%}71`Qa-P@#qyBT%LXsI;3i=*L?G-GnwPN1v^I2plv?4!O9m~XLU*w?nJt}Omf^{0%y!5}-!38ZPKtb! zk83rYx4W~tla#u*yW?4|aG1AyOSrRQiWtl4s%>p>IA*ozu@)@&gxL1B|3K||b%0p4SiN{#R|*%+#57(8TOQ(|7&Q?!UNvG`Wr?G{ zCacCwRl%`qlEAF-y%rQzl}y6SW|>$ksb(=uRn361foc$L8lp0#) z^o3r!B5189+HYgcE%}!!LCnl7gl0N>q`RK^OQ9hx}P*3upx0C(#d2-f$4jqjLF2_d7~|U zI=)AeKn1oO4jEHsz{d0_c@lY;Fo0p_xG=>#jGriq0ba zXse;evNUdkJ#I#yxm-!Q8p-yFeZmt^VvMXI2uGd9(5x%{-7~oz_A48H+o)VDJw}e*#pnQnWPDv(h*CGC})K6aR*bPxFs>?7mX?CnUN*Is3!> z-`-zF|8LFz+4ldYO3%VV&H5GYUzCA~k(!N_mIP)N*UnUtD|Lx;1-M^gvMgP0cFU$X3 z`2QyP%gJB#kLkaV{SVAv;nUI5F;UYou>3pfzr$ytr~gU_>(^cNPx^Pmmucp&TZol` z?W^%$3I0pzFZWDeSO4!0e@DdnS8D!J|Ev2SlYgcCFWFx@e>eQw>Hmp{jgf(xm67Ft zisC=P{sr#;q>RjL)btFj|1lW;zTSUB|2Oj$*`MV8mBGK6zpVZ(_?w~s7v8_+aH5pU;ldl>G;oa%EU&^!py|{kFop@$=~DS z&kd*fU#bn`e^i^Vif3xUJj;XsmJb`bu+0icP`lF(IqbnNr#n8>x z!^w`tJo0>RsQDE4MXzlGx~zuhOlzppD(mHNd@5eGZN+}*cKx_(mHKg<2#%*6C-VKl zQspxBxjo};EGTIAX~DCZQ!|TtX#8gXcK*V^;q$2`#T`%iy%{6(^YLxlq08&I>D}7A zXk2%?^2NdPz_aDEaqZ!J4=?DW{pP9ha&`H*b86*W>0PJm?8SzS^KsmrU95ILR=_kAJsvf<9uve2=|_n4z5Vt15yA5b;CtW^_h(KJ!`k`f&F+G}r21(lLBFH15ZdOL?6Up+P@TPrV) zzn*OSOjpG5N8l%!3q_xkra~$_5`GQO!fyU3UFqZzcioWriX!^4R*=A@F#S44gqbcEUufAj;w z8_1pLVXnc1Zlt%8^yhC5Y}iAu#AP3?7>@U?fo9b5xvB{ zyzL@FtSOS;`-PhY?o5Uu$b{oP!7&h`3_9jQKR!Kaztdt--ld5EBIbzD(|03D4u?~U z@+s*HIPQ5LwnbyPM58@N;~8Xs`z@$CRR#ZBSSQ}k_9yUuj{6)CO7@SF5Z9cIVtAAs z?qc+WY;y#m}%&Z+F z>1OvYpc2COWj`(6tSxtJ!Sg3H^mUN40gr4j$`=gdS}5WvE1f;PDPeOzgw|}hK02sH z!D$a?J6paRGxPzgskjlKr;EL>W1(<60;qZ6J|=Y0Peq$ODN_SG4JWb-_w;yI=XsYu zj@ZxmESt57r}SlS?)QkZ_dL!(FPiJ&xPr9?-EGBf zQDwQXDhQ|dYW5C!uSxEZT>)N@2Sk>1Xwg`55{?kg3^sgctfps0`-BmVN?R7U#QJ=E zn{oq;aaKY$2AJ2naSkVlFGVv0Qkwe4r1wHLVECPO73kO6?v%bgoGVU;-v`t?Zz5?@ zUwdRAX;%mkYcL`2wO+qOVvNC9kasK}Z4yh)voVFK9|oJhmT-^{b5@T05wznko=#Mm zLAV{f;223_k(BWG|d(y6UpO8%m zH)FC?V>*I5hDn-(9mU&{u~Xh4qR)WhsrW{H8f9`E>(SmpJi9K=TKVcdLYh;@l}$)GX-t7ECTtjV%1Yjz6|^K~?7dx(C>ec-#{y#eZu z#JIo%5S-+90wB-itIC1fz95|vy8{`3TkGP%jk5ADS_@p<3c3z^lKT367F0Oo&Wib1 zCa_!nFxrK;sBJ`GPsjP?=ToN)H!h$+HK(erzrwP>9|!yJ3)Y|`vGhn6H`f+^f%)`px6J}oGN)*N-=ssQmG z(aR&)2Hg4+^xAY8v}TsfaT$};I(P<7)o#mh%?N8$S6|EABRG0Jx;1yGul*mdG297w z(!3Hs`L_Z(`TeSuos7c>)UZS_dj#<4E#HzQ#JCWEj&;)nKfhJ#i< zAt=&ej)kRU26eK9BsI)x-;hJKhOR|w3l}^|)u)p}i|WB2xLs`ylRZ4@e|te^(W7be zYe{fR@}IiJk6?~g9v*e3U6CLkVXY0|dxG{v%>Zbp;fLEzyRtZPSqpjHE@+0P5y3-1 zW5BH9O&B4C2qWgi&JvgcvPa`U$zT7Z8l>P2^Q?G}WU6uUbwA<7(h=6?&DuSjqDhwU z1e=KfFkytqAM@~jOb8b5V7*$l-4dY@jZS;cGp0Gk1GwuQ6cil7o5Ll6jE!Nt;W%dT@)?r^a)*-L-nzxqF5M7TY`ZTh0C-`Ft2)m4bSyLbM zhTm$A0lDZo;2l-?YWV2L`4uP1MLk0P0>aVk=2Q~zdHzOKCO%r=mW_2)7T>B=0O_3& zj5Etvx93zk;WYfgB=045d3Fyg77t9987M51A9Ll_M+n48NrYr#oa&g)6rGsyCg{p& z6zCS}Q2@wb46n3Clz78wHXNRAphch8yxaAUlthhj$~jA6FaF7%^x`u>9dM2)%@HjH zuqd27B^YekXClV)|AR8hi~ z^%;j_e_UaY!iT%U!Ynd^t$rW3FelkI5AuC9e*+CDXUW0 z{(byuQ;^y!*WpgA~eT8Aqn1euCQ1`hb*4~NrJ!6ZqOuafY@AS6R+`xU{^%lI@Z>OJj3A5~gV*mAtF9ueGGaMORYe1e7wq~h|^WAxuK;IW4VS8tX ze-{oiw-FnnSRA(uEV-^Q6Ii5zJ->dQ^Ba*LW?^i$ zl+tUhS|aB^J=$a#r0vsmui~D&!Wi z0y&@eEI7W-15xa;heuU+XNy?{Zp#Zat_~faiuYx%MjAuBUd!MmI1HZa?9ZikO!(dl z6=e`$W(!PJX>&2=8Q8Go+YVqOu(f*YYdPcVNGma%_ElXK2c0D7{gP{N-QrVQ>pJa* zbCxJo7+js#*?^JjsdN!;;3V_oerb+}-7SNDzp9E|DRL_6W0ZXBeNE}M+~E-JPv5~5 zufE;am2YG(7TN#GZwgtrK$kkkFJUIW&kp{;N1+@i!!bVzU-4jqvd}H9U~+Kqd@)6s z_;EKz08Ec_Oe&8@&R9Ccs09I*F_h~1GcHUw+>b^D9}u%W=~o0AyJ&)`2zD8rD5?q! zVb!o5rjAw*0+G-5IbUY6Z72OtHo4={atVc8@*fmQ_{Z|dwSna=(Z7yP3-ImsVt?m^ z{TRmeG4c`exl-P`wb>fkunCUG2O?9-qc3u;(?z8YBV2Q?NjCzIb$00hz-qU|BNTF` z`(BMOCJ62o%oz@RQ*BB#m!J}bah3psqhb!{@!U>|iL;DSRnFm|b}Z>3LM~7Q^GpxB zURGKzkElJE=Mic>9-T^%?4w1`*z;=$1?Ck?MO9*0x%sCX^Y*loPL*Tg4CuIlBg5?7 z2j-NNsX0^AighN3)i0Mw7RlX03{-0ix_+|w7oyGxPNSZfO- z8M8F+u;zQ^PMG;vjx}0_SPS|kq`xh8Vj6>D-nXZ7Alx!N<`!;wW}T|Bq{Gk>%r$#N z;#jrU7wNr6=t%TZsE?~$RTTq+a?~N3>O^qInw<;RKM1L9JTV0W zoYp^>x@q0_7axYjBeUtwu%UlcXuD@0#qJ z>%3{LVP!2F(*?irlpY_Z#%}QNk`tS`G}Hh^hP8U*r}{7g3H$xebx6Wag z-%DQN9JHCrIb@&n97{iLTe}v$eDIDxA3dKEySCfooD&-TaY-AHyd)Bm8&)9FgMNT5 z5J{+fO~d*AfV(GQcAzrb)v#12Wa%6DS^;1!b!Z6uHREmp<$LQ^aiFxK1g88ze}kY* ztRjq)piJv-pbAS=B-XoRi9kb0u(goE))@|`V4;;ZWxdh=l+)r*ocx_$i`y3NQZu|Q)H4#yXpoxP2F1ChD7_cWSuGP zKx1lDDh~^T>H$9kol`VS=}0{-g&I2XPgiD0s#h%RrEQAeKDecb^<->4*s*!qt?8PR zv6st45=#LoNs~sP3}Pf)Rg`e$aM8WrV2g$2DYvN3x*gg!uLSvygw%H&GN`62ouTpx z6K<&-GNuzNClV8XR9la>yWB!jAlX3nU9#^^7sq<%Q+T$X+&N{`7b<&iucO@Ymhcat z1hx5Y$BWw$yN^Li@m1y~I%5ynRm6yzlv}Bqgv{>SLd9^FL6y5|SqDlG#t~N2Ej`V4NZg)?k;jX=e(rjniUldGwer|;gG0+t;KAdak!MavI48B~GUdiu& zy5Ag6R>&7Ft+E87A{&Q+;dt0?281U)azu5dOj~n=VUNFylj;+Nj@y}*kv*iMaL#U>Nw+)7)kzNft4;OKyw83-IKK2TP{FQfDrIq1{#ioC4n zArVS;-@6<`AGE3{rvWNB8s-o1$Qmwj^=Z?2JP?HE5rdl*(9Y%N;>TxImAL_5>7GT` zQP%OWGbq)SdFQ`HlyBabR)y;c70FY185cKe^!}2&md@Rkz7K-6)3!}dwwtkLC-)gLwPOQWrQ zEs}irh~zY`tZO6g#|--VPk;b%i>DAlegOeO`}-vFHabhAca)P?bmm0p-O#oKfiyJG zU_cn$Rbu4lbR&@xRoi^8-?h7j4F?pDzE{Gw^D~Wbv8cyW8u-;p=_J6Wm#BrZsP9p> zfX}5jW;055w_J{D{srX2i4tgw1{e8@_dCO)j$YnSd%`Lp<{sUE`!_)<~W}ltjGpNtN^qeEN)#yyk z!X?Y0k7NzCY)*jd2vl|!?ufaMcfKL?HHDCqB}2WNZce&e1C0+FowC$t1qk@l?-OMN zJUD?KxRMqSw_Wab_H{Eq;yVKX5@hMm@#jPwXawSLB9P_&K2|Jf3TQGwEU`pSMoZXQ zA^eTvc^!{~!ut7XkV1qpXhm0rifwEEcyit@ME-Qxv{j+Pj*p{8vRx%?RSrE~6BR56 z)Xys>SRLDw>IWak_pZ!^%ymvKrUlcZu+m3^=93XZmm6q$feqlshdM((KusVo=70xC zpY`HL)2b)Jre)Dry|fB-zKoDrWv3zTXtD@|!hQogcHmxU9Ikxr!Mlbkixs-&2M~+7 zOl2B(?oaI}HjSr{M3-&9%LeLm-BKEi@Sr!eJ7T1Fv2jhaAw8g7`jIJKDG~^DnDn5! zWEkP)J`i-Yo2uWzxr)s|A__r!?7Q(3#S_M9;rmRe47RZLKQ*I`8iq80-fw*|N@)m{ zvgqji^r7q1e~!D(b0XZ0^D@!|A2&!hqr%05sS#-qS0n}Rjl0s1!%%Is&Sq8?Po^D( zig;xDCmBK*UuOzumWs;)cV5OzSJ`%s&hB$nl=(6i$#xIzKD2ile7kEG)k+z~A)~SZ zufI3|VF~`^{3w#@6E+Ol0MUDYzf9R)-gSb;<;k_5utMEu#na^Sz8=GKuqYe`O}AAq zd6^5}?n0|`1%7bhDsy`(^zdYuGN*ir#u@SiwslWFLBR;IjtfIiAdtlPI`-j&ZzeR( zrNxi4gd96lLuMh)!k^%4_1BoA&$t&R%3vGf^=t{mSVJJS*Y%JZh2;SFW&zbFJk0>0 zR_S`)yLwyD#G?B-z9;Bxu+==b7~@qfuTW^#rtfG^etHlgf25FhKRKXGuUUEx{`>H3 zi1hCtnh@q7>tZegxPSoNP%a*?i2;{=8@|uGql2)7X4Q`w#E+9O8<})2lpY%v&P>aa zjXnxgN&-SEAO{pc7-h74g~2*01nIdvVSHK-DQU-b(nF`nQ$zt7HqOA|2h6ci%0-$$ z&6@(z?cd&wWFt?A_C-lzC|*XjE~+LLJVM2{VBw|*QtvL~LiSW=@UDJm+sfK+1;G#U zfx`^@ECzHDb4Na-Uff1+BvrD*7ag9mJ1CP_lsu3j@`}+t6~Z-ub*wpcvEIS6V}fz} zq;U9-O(g`)tq9#SOUdw}CpRUlP8{9)pP<9I#AMMBpv^UM9D&m@y93u4r>C*IK0*A}D%2 z6t-Zc@^npijwULBso|sIAx6F%$yuwMR?Kv{32wRS_}2E0)rHG=fC|H1RpvLi1}!hH zg$HbROC)IyY>b~{Q)-XY;y$`nfjbzi_4Y$H%kUPV>JLo?$}^IAHCPJn=dCjMcZj6&!8%JL9*GfG_UthsaFl>P)q zncUM4zF;PPJiuo0jONL@O{b|(>et3YkIC1KM-SbGF7`Zj+KIN5_X;D@HZFlZ55D-x zr02kz`Ak!*2AQ$zJ!d=ja0)=|*1@`gE@C~2Uq*uy-o^p|#PHB}5R~BM;O5}iP{9D~ z4AemksQ97Nh}pa($U)@1C3&mHq%!qL=JoblHm0njD#k>@>~4eSj?(O~YIV4+XQQ>) z%BYA|hL6aO$9CX_$O%&+#-k?IcX`)|;_JEOA%kUB#+<=`BZO|{DD`k|l&*!T36M$( zEe+pGy;9{&wU|UbtFoTXlPlL$9P+|Vov^EL=AF@>SRZoanng;uh{ zRu}NFz`vhy)f_%gi&J{x9EB>smdmU?Z&S{DFJxkxK>j-KQpNqWKfvrny#98vnYKvN z9bF>pyt38vEMnGhGv<1}ZL^)Rz~d&nGEmes^Tb>GjaG(h3ZlWe;TDIqg>ZKRx;Dcl za&>!Y5_{)?gZn=HE|eOqbJL+K&0il$w`TPM(jWzy@ad7Qv}VI!@&5x;K&-#hdT?1y z7)&LiTC(hgc4DUlwr}@yplQbKQvh3lK_E{tG_%|DOl`&^FbhVPa(sE#`mC*4yRu%( z;<8Q&$Ln)&jxXvu?p}zGBX7ZFhU!YQ6rj*rQ5a882~B9VrIXqYR4D31rE~?d0s&Fn zo~!ehPCuc{b(xR@Nl7tX77(Zxfx_82byzXY8+HA{dDr!=&6>BUarm(51})*g<@bw^ zE}ik=vQ65Te|=l~1+NUvymiIg*>^h6Fte_|wsC56&y`yzthjm2>$mw|S^2tl?hNo0 zke3|-bd`|_eXdp*gQg0}=ru--qm4HizcBKhCd{)gHkZve4KqzJ?JylMy=9Ux5s{HM z37l*&3CL(PnL6-GD%h$H281**vL?nvSQ!bb>4@olsJx8xk%$Q1yC1PE`~Y>}#=YEn zQ08Kc|cuimgpQVZ{Cxm#-hwunIOy}&=Enp7S0e@5Gu2QSZ93Cc+$wB6l{UKQBZcJxWroG3|p}klVyo#$(`Tr->;q1cHzA9%u|9eGVxEELC4$#z+5S}9b|)|_7KfMW0`HJZMgSQ)9=iiY@eDXo7HX$S;MxKHV{hK zBm*66R%-{@rn<}~huLhh$qs5is+eiO8$j$`1+W7;zWgRIv&O5YEV)o7GKHeN-9h1J zaJXVYN0CEyFpduVyy~!8vy?)Gbi3xduTGz3w^+=qMFBv*@5ZVdyM0-@J8v^B#IL-M zu!^>#T~uLaes~aHlNOc~o~Hq(dVvZ}MVA=R5WS4nc#AbfTr&_CUq-Bs)~leJX9vm( zVB#adC}@nF3D{#?P}jI*@ubDgr#FzZiGTIHaq`QUowZ*3Qxf6Dfr)e0Z&14W!9Fqi`s>wf7iai1^?RdS>lOh zqejl0J7&~ec68>LhHZ&EwXTo;p`F7`crAViPks5)*K2>f`}18oID}! zkj++hJ@fj)^LJtGzl6@W*e5Xe=lR;Tn*9x2L>B!UP&qpJCF_Ry_MjY8giZU3#6VUo z(viHN-VBAet#8{u;XV=A`N{&v3WbTF0%Kp)-&li27;j~#h+rK#7v`-ghh-FbZB+j!KN8z+vbpV*ujaQod$7tC1tK*~-X1oPua z^W*z!V}EO{b(&#`xK@0QdrsVL-frKA4l?`99oD_}H_^M+BX(=dKF-i&nrOYw-fZW+ z+(Or*?vu*r4sNC$r`R*gUkL1}rdX3xLP20nN@hi8&msxeOP@>U((Jh{#hzE_Wq*o8 zO`hZRHf;0dra0Aj1&4NBwrT$t4(%p!T8fo{%p#>RU{1=S;p_-bM<(dWI9B8_L$6qgQ zXUeMqm|e+{0fQ%U#O=!38D+Fnm}pxbjxbwo3!{kN&3sm`2; zcCaJBXy?(5s={+v;N09qj=@2*HAXr>HW{E$#$Uq|wv6e(>r{y^gO2k&%h2eRLFMia z66_VyzH6n=Th|E=d8NKy(3$9@W<795yd@Wxf)dB_A+MO~#c3KM zi$^Q#!16fXZGKePE|Un3u`IrrWHD2O3Ai~!Fry_>cI!m6ovmY&K#%FFVj)-}UeZ7q zp3;M(^e|C+;2Qi1&oE^yrHG8nz$GAe2RWXS#Ee*m(GCC0REbk^)F}y|PABqI`L3i( z84$&aofFBciNVajwqrkbK;5g=ozrPD8+64g?j1cMe2-edD`tC)QxsdQpQoQ!<;FUv z`<@qg$z@0SY9Cxh(D=;q3`+-jXpiz37%R7{*>;>EXW%S^!Fp7rkm3ruh|3qUgcJoU z-rzIcaRHKnzmUsUxN}_fp-Namratq^|NojQUrn33Wd17 zc&Kz%RZ(_U^g#EovF{7vo8qsN#R zJhuI_72AH8=%6QO0BqWckQdvce&xb_N6>ffMj8`Fn z+HtiZln3V^>46HsHq4g`jpe9}t1`|;v&l4OIyY0CA%DeKuH`WevNEzHv4Vtg5D5;1 z0L&~4tmNQ0UX)c|V4$p<*Se z&QYZ-2^UGN5|Iv(9K^yEQjo)(*Kp&M?jVvEbPX8So`k->(Y?S~N2k0HeDbIs6k5Gh zU%rG)P%0}||KSfQuVt?klT0xbrS3{?n_+ZqTjr?o;J7nMZI8&xhmsml>e6nWEiX^| z+$oPe6k_0uLw1JYUejJ}P3&K+y+!)tih_6F!gX3Zcj(eO5=@**tyOFU+D``B+ffkp zM90;tg#~z~c~QZa?0J@zLQaX#?-|N<*|MDVPEzFDKTlWMioHS_Rqgm^2JB?E`!9`c-@)Wf! zK&{Vd={B1{fAL%hTPc6&3vQ7xh?aCE65dFzed+nt@nXS<>uay=`N2+X+ZG)>T3!4ucPMbfqZ4Pm5Y2mK`P9QLCXujjX)kip_16p; zD{+Z~+9JsqA36Aj0xI_>CC_px-JV{oXjaX(n7MYM1ye&*b$Vi!4xF!M+j_~2<$Lou^gYsVHID?Mgis?hK`m-?CMgz9y#JiBQ(+J|J zux^5G69Gr_f9T;RumSs%sfjJLch6i%JV^UC8F-Mce+NO%)s$z}W{wjUhzpD>#TCYt z?iGHCck_Om+vd-==6mve`I+J{!vuDWG~O_qy@Orix!t$lykB|S^p^68a#k@j8GMlP zoSNmU$fEuu!Pu42iz5aljU!pXAOl0rE&|Wd zo3~BI-D|Rx_d?6IUd4NK@L(yl=z1f?gTR6`oLHG?iB{@jPiI418nOZ(X6FDPZD1h; z*)%6Q6&v+zN|`FMEUsbc{Zw=1MFlGenYeS=%lKGhn*Y-9JPS zML;Mg2Kl3U*x+6@TQxP9noZ1HQ>%$kxNb8YF|j6MNKN+<8uiv8JO64noOeE+-mRu9 z&(!9iR1BNZG0UXX0nPpcv)!P${N0H2$q<~c9Z$T%9ZI}L2K`b)?n=;@$$c=ey)Y&Q z&Cz+Gy*Nhg@gwcwTs>E-9PSw6RIY(*-);8CHiyYWt_&!UQvFJ%f(N+Df_g%@z4 zufZZXz_9Y@QDD6w(t@Z|ihp2en%=vxPYN1LF~H5cd&E4tVS#Q}sBTof5k61az&^rn zW82sxtgwQAmi>x7&w*u_Jk?%yT}+~Y$bmvGKR~{nS;aicJSsgV?_>@!?=bS~%=^qG znHeAtX2?9S%5bzL-lXG|Pkz;IGgR>%$*)wq#Zbi-nOu+<9aU`5V5@?jyx(r|rgF18 zmBST!4j1Wp`no-4dlgC+aGLxMq&6XFEMeNhkPI;enNIoz9BD=P-8>FrB zThcr7C*(MLOgbT-C10^;q;F)?LTQnF54n%MN4ihmK!iraG%}l=A?52brCipo)TT|(PLs< zv7ideMJyg(4zqCB5HuJ;bd;+$kXb5Rh*Y=;6=zWs9&TfzQdsF~p*!Cy|D{y=yh@@a z(NgL2(2a1=pqy?&sjZq-l__?+!jDP&EcVfWc`Hs=ynyUpcBBodnK?sSANQl5lKrf&b!_pD6bi`~ie^g&vzE}0Ch6Y2kfthP) zHIQwFBL zgPl?(K!q4gVdTgGiK0e#{Gz%*s@AV+-ESxrTMec9zy|nw$HdW)aEyy7W>ki)VOD}^ zwN2b3o@V$zFh|8tMJC7;iZQ0YSTFvD*(z>hc8P7wYoa0LP?wa(h+3jM)TdNaVR4KE zX(BjEp=FaQg?h)x7)bQ1YchjSgG3~P;314#=t=T~{-i`0NmStmGER^j#4ps5p~7Rr z^TMCV3Gx*=EBu=n@<_gLt*}T~BfLO(s_*7SyFaKK3Qb5y0+k`y`Unn^Mr_wUN$dsy z^kP2zrG|Os(vXxN&;<6|S+L(M$d8^>$8O>_iH{f`F|#5T%%Vl`muhW zcCEP9xXQfJw$`!6xyHT5v&v@_Y=8xy)8=#dJWiir?`4v_y#&UUw@b!IR^(vH`ci{M z0X5JZm>XyfYzy$gz_|bkD0$lurs1z5eb(=5&sg^R%O;xcEyPp1CTN5fpxy#LXRM3{ zvP#lU5yB4J<;Z+fb@7Wc*0$pzc%`;XJFFelmf=2M?%w^+lLro*A|IW4WbU45KkZiS zG3{|}4!A}$|E(pH$x9b5QaVr{;R5ghrNcrsm*);T4tkhtI6Q+p#u1w}*JL&$zd~(n z3ld%b%>!~}1&Y$+;{uB1N;#i#wG(vN2TB{;zxqKiW#Hzeo#}AcOJHbeU;4NBGi<(Y z+0MzEM$Ufc^(S{NsJ?Mn={D|=D|B+#>W*1fXW|p~4XwHNzK=0=5Msl5@yLu&|)x2@CbARx*4%K zuI#+L9G8F?oXfM{0K!U|X7+wiH?l z)Pm-Mxdp8S+Y0z#!MOrbkd?R1h%Lq}W08@ep^)|-|A`i$>*)m8@2IM)GpZ!CRGeMk zSFFeX=)PgPiy~LwFHB>8-7uo2TW)t$&q*EZGP=!->vK!%E!C>|z=mDgOWIwy6%WTX zcv)$_cBrD_)V_WH_{R%s#rXK>-y9m*`vXTDZF4x_)&bbs0@}8DFuwty{}53jp?FS9;aWZ4WWM zNjPk^{)+hUMl$Eyk6Yh(y?N>EcI|h^=8d^=dgW&y&#tT=mb3RPcc}i|yPx?aqkPr# z+84O$`KC}}3o|mOaqzVhj2yL_UFIVlNIcO(`|PApW*)`$q@^jfTN^N|m;iAj?BJ%VvUE8?qlnZpTvy0V{MPp~z z;-GKq>h%u}7!f=8ujbXuzkd!puv^fM+m|lAYj|PLa@=-w{(VXGn)Z!$44=$+aLwXT zvEhDO@BZT!zclyv(|>r^)H0-+AQ^SQ)5nIGH zNOPrDX@ey2m?OC?Lj)vBZnuwJPOTx_OO<&ch>Os2%0*DMGUf&{m$Z@%ge6`v@j`l5 zMm6pxFe{a*{Wy>eoi^ljT3=M^&H>$ILnAWybFGfOUmMB3apA(nfvLTipwDxlUtY9M zEf+*V5)=?4(lz2Wl5o8=PT8nzvOeP6?AoCmaDC!D!=L93CZiD?Um@2n84W?xQEK$+ zp#^^feaUF9ztvBI{v!W2{}Dgy$KZnpy+z(5UdBsBf$x6}Ep+PQKzGUQ;2XR3N$Xn* zHi=>;;3m_VdkE(n?CbAb*6PFgMRy&4>8~f2IRap$e0jKh{4Fy!zQjZ?Y1)NPH#SY$ zJa*Z6x>qy_vU3cKkH_YA#9)!e)@&8Jb1=r&FLp#chL4dG>sZz6W&N3Wgp?T z(s$7ee4$94on~<%7QD#C_dpSTIJ$#G?|2bruIcqCHK8U!YH zh)hB(up)rlz~bd>EBiTnie=djY*6LpOe^y_a|)a?{Vkvh1Goi;umKUej!}d$_6k?5 zV*roEyW;Vvr}MJa-+8so|EooNq+Lnbvirb{9RtozTsLHOb%wxOMe$I|OC^Rx7aoLf z10~ci@gsNW;_ul3ztp7SpaMd|z0`liN!1o*qI}R$Vqk%k1{FvL9&~Fv9*-BP;c0v$;Y!Xxz*`#BG*;YZTLKHX=5g zcxoM(Ln$Xjf&w^zA=>3F?J3`Ii|VAF;#68Q@C?ENt4Bk1zPZ91)K!vy?;8e|3w9)n z+IesWJCeof4Gb~{ufTxJb6l}Ob{Gt@#0x@DlpLZcu?C|t?KZ+g7$YK>H8PxR5G0Wo z1%XS$M0X%Tu>$uOf`8b7i&QztA2u9T3#q>jRbw!MiM8fAy#68`M zTgv6v_vp8KQ9QMziLBH~th*91S7#}2M|Toi(jEd$S}0iT02aFr8az#V5*MDtM$jtw zAGko=F;%Gfq)n9$W%KGI|?0A`dp4%=wFFkGe z41Xl>D-Dm}hndH?O~PZ+!-i+^(~RW9P9YyhgeE*rSjnv8)=5$f_ZNs)4zh*v5O#z- zL0-Y$CqKw;mAA1Uv459MWo)_p5W87^n|()qpA}>x@dkkrc>~Lc9D=#!kOb~}kbs{A z72aS7BF+K5gNdUrSW1!sfz*BF0I%AeF}{|rK(>oM6ENf}hyWu&E2%Y5)5nljj7B;Y zUOE>v3>NK#;`vkw6<&^rUCp7E%Nrv1fu75YvD8{fx;t9A3Ux&vg&X8HmsKhH|$fXw(kLs9nfv!EQ$0(-L zmF#o-EENL5wzRomB%p9HF9J)-)Jkd(=EX1-OnwlRm=Smt8u%boK&~^nEND6^7CNgE z(A#KKPjL5}hkn&bD7#|7;h-_rsJ>v7dQ<#MJ?$3z>Cgm*LHL6)-Q)|?h51%% zsfvg34R|V^!_URb`Bp5FU?@K(sBj&4RonMiOAYk$QpFn^Da3^-WR@_GEEVn-_K}wb zDL@pV7YPbgq$nLi zGg2=t!?-)+6glZ_E6@OkzTd|gww5@5gMT=CR+}PRfAW_pC;$6%BlAeR{W zo(TSbdryQZ!C~eop7SC1A$>~(qX>+{qmN^VT2RH+>%%BwRecmTqzF7uvOs2cVUFF5 zJ_mMfmY!O0c>`G@p6)%2guWev!7?kchD(V@GoJ-7<-&6(xvihkcW%gDwjBD-qcwDb zFQ`4e{+J7U$tUD2VV`Ap@Z=G8BTrW2HH0kTa29tzbcF|QW_cFO#S{B6Uyj&vP#(Qi z^XJ_69N9{jFN3;m*;_Sbp=LCfZb25F`QY*kq49WvzEW1G@9omJ!=$AUe4=(-fx{$8J>YxGE5yUj_}^C{JsnGD8H*v=4dopzQ5{QWJcWd9_{&4Ro@YL|+S8Q)~APS>m+@j7Dd}0O{aQUwF z56c*y`Vf2LI2ynKl+V4Z*sbLuPcx5;cW6QB&*1F zejoo9Z+tCwLLq^i;y%`%rW`feJC>tn=R@KR#OCB&${{&5`uiM3W=m{90MqwQ+5-O$ zuz^5NT!K_+O|kpfkeXqw3aNet5|1wy>L9o8V+Es0?ny-xoYQkSt>PrBPnbr>@H zM8qCazNsn8xYbCqz(zA-qiS}=jK2C9jx_^qAD|{=NsrJ(H~!p>>)jLGbKH#ETQX}< z_evNjtCr4qO33K7)4C-NY)zM{qEX$i)4lU(N_y@o4B(cg&Td09Cwb(6KM)&}Gc||A zqfJyE0A?8d)$7!TP5@6{U%Fq`mYaB9SXo+GVU-W+L;ik0*w2u2$ zj>stUiZwNR){MWuHX}EfF=}Xk?XR|CPhMsJamA6coN0PIWi{a8VLf)2fgV4IY`|T8 z3>9VmYy8AEjvptFbB*&fW&AAgrEGswe|xEaC|hf)wGZ__EIca7Ml%2s`CuCMaDs!* zhTUMWAlV%feRDH$rqY8j5laW|0X#4lwNjZLs7g`1rLwLwQTgRar6nzBI;q#AN1AZF zx=}Su=cmilUDG|YGPrmXitFo=bgz193v8a#?r{IQvv@W3-o57yO-me{uv@johA)oa zvtq`yRotP(xrep0+P}4P+NTqmwvd8n>gR5Ke*Y7XQ+^u@W3K{U@S=aHqZ%zuwkFq1 z%PiY0*HX`7?_v8E9s+8juQ? zA{DMfB^v;#hk)BzI2CjYn6}~&Dm2l8DO3+mrq3^g9A01JpkJkRysk6a)fuIW@$u8q z)P5MqQ!=Zid$)i0d(1q4dA~o~NBh=1rGD1Z%FhFZ?UWMjWguC;?ZLs z+XP8DhNFImG*@Bsp}r7$|B~vq3OhU@ByH53?zPDN-yJ&ehr0d2M zF}hMp#kKWl7+}RTi=QRWvZc_%$D2Z-()0jrCB<%Q2ww>mc9M2+(C+aEw2Rssd+x^G zgspJMl1Xb;%$T}*%Y-JJhrwsU-iL{DY3|Muw?6aqfhV@=`^>*$^8hOjl!2c*h!o(H z8bigS(qpEL%Cp=K`DN*4Q-@C!9e5bIhOd$9GoLl>=lA>Gmj7rxCLcFm5Pmk9GAtQR z6=Z}{HCtmA=WEWRPR6ODA+t))&2Gra{i@Mowl$cW&BW}nQB!6=C_G$Z)3;Owf~l=l z**#Kuv{x$kWaznS0fD-WdK(IKd!o$-WNv2-HV-9tjzK^nTh53W)-MprVlmK$D36k zR?0ijw=T|$M#04^(PJhcEijjU!JlFrF(EyIVoyc6+~gPR>HGCA?c0^Ju;ar{Y~vFu zbN8gdD}*@FnPJ7wS-p z`?9!z4Jtuv6Wi+HMD{fgak{L;VRKo{b_-I>c8nC_kVK0CPc$SA#6TxP=CRe{!bul) z(Q2jw_dN%<=IstyDyb6d#RicP^OZvDL@Tj&U{*Dm?GfUbh_<VX4F|N2+$>=riCuxiEVoIfZPqiRoHI=~Fqd0!jFHtRM-As(qtUKm&% zAU30EZl6`A);??yhe?1HQE@e?W+qsg98J07dW?gqy2*0Edckh%Zz^&1&oAj& zYZ~IJ%^%Y9oH5~+9{{awFq#YnMpK^I<#zTm8Qm_{lS4URpUwd~(9KpIB<%)cD$g%S zF+eysmG_CI7(sISb=5zSqXH+(l1DSM+=~+2;1oPwzMzL8;`30UBYC|(-}*kd4~U=+ zRYoN_A)B}8vh?Xn%O9mvNu2IhR*A0k>S4D9fb`Dk58nfmqr>;tMHG5aA1Jz?k^9%dde~b;-_u_9a zUGvJJ*_8($Fq3YISS*aULjh4!dX3(mq7bRw0(c(C>F_}^F(I2V^g z8m%*}95{J)ht+0h90Xe^Q3H%1$+E*KgVSk{Bch}Rb7H$BoCJr@r@!~@%Fc;x@NDyt zxt?<#^1TOpkR#%9>S7D7-R8vSoY?7gSEUHwGB29mW(TG7=~^l(rNA6i(0>9#4J@%T z{Yo1(5Hg7qpd8cH9Z!q+`89_pZK)4vXM>{#)ZAL4fp3vGvvt_qHR}@(l0G}emkwFG zD)AltC)_0UKEA&1NosZBZ}y)S()aamHcv|QQ-4IIM@h+`QG zeUp*yuVPRfY$BfH1y&{k=u91pp_~}&71QJTbsSYXd;YpP4489XwELxvhf?peu=_Eq zUAlNJ8@Z^zgP43$8-=&)eRQIG)jF3D5rXc%!hVr!bz_F*kkesOjEeN1ePW&bLv+w9J+o&W4K@|Fq*qi{%{4<;L!2exh=9 zEnsh=zK`!m>i?WtiN_;jWH1LO zunTN9i#J(zfhCL2##u-nEY2Qc2oI0mt?Hi9$k@EyytD7UJ;$=E>vq+>x9WcC*IixX zRF5i;YhKj6s{Xm=ZH=af9b_3jpP_D1ZcuGgDo&~2C)B==qr*B50iGIkmon?U0y`ASEK4& zc`H46zKDVwrG0nt-BNpSCT_(y#U8}>#QrY!$8RW}`}`8#7W?Yuui#(ri9H=ZH-h>= z58WRq785U3bSao!iiqN@;vB4U>88M)ifKqtNC!f|Y#gJwKf-$S3;rm3pnS}Pya#Xx zLdt{C-#hV&)2JAzH$qu3zUoqyX}TJ=Mpcthli$QNtC};K@^u1RQQV^*EZ$XoxcD!W zPw+=|&nTbKMT*ZApD#9`;)>!f$at~%-C|`iIRvEv?sw5l1)t0Dwj44bMm2mc9Tz#? zWHvjqva=md4KymF$zd^zH!T@7V?Q*JS*Ara+Oixu*^uVX#)H{78`4e|Ivh?imy9Ch zq|+;P1Bu0@P;V!oIz=~xdI$y18mGAYhMEfJ>&|zbtkG$A?sBroDL5;gb572gS@fIw z_ySRW)h3Ms^_O5K1B30Qts7nS^PLiX{| zCWVrA+GgIMSL(D{n1nOfk%nlejqn^f$>AEEwnppF8Hj%MdR?OLmX6k1*mqM+eWCl7 z8b*Bnp1Yv4rb>b?oEwO}S8!upYuE){pkm*zHXiDID zC!ibV=7&I5J*?y&7T3Iw-&1{(@<}T93iG~#v1BST)y#lteaiZ@fsBWlgUW-dhjp{+ zH<*7?yrX_Y_nzWC<@kRKveab#9$%AP;{TMTf)o{+i7vvlSFYCAeMnC^yzqtfIvsb(T!yUIlh$D58<$k#>+(0NPO=P}mwF#oR{!L- z)0l~Ue!BnuPB3!XeWOEneRKHM-C*3Vk=Q@SF2^p#-fHQ){5$sa%v0Z+dEy9JhusXa z4}t7vl!Lw})>`VB8hwqmK6@?Gq;ImW%~l2MI7gM5UNfK=&}`7JPZ>xbu&vKIsyUkd zsrr)s6RXaQ3|T}8oHkW@V8I(r$_!X+F0vHEs@q{U(+3vn`(dcG*`+Dqr6l=3zCwK0 zR#^c)g-j@&|{HM2 zQN^T!t+4bP#tZ=qr%{vLKFs~PIUUoW>(nv2S!PNs=6Pu4EThyEA+PuvHK$jWyK-rCCtynvr4lvU*$|Ur{^EE9JD^{e6U2#TU)G*<$3*S!z;PJ&Hq$?DNk9f z@6&tr4{9H_Jdt-=$Gh{zf+okUc_WT{E%#c#nfHwXb*-aW*`i&m?=-gLuFB){3Y?Bw z-I83gUt&oCuhb~a>f8*yQ2xXR_Zi({=kSQTgXE=Fn%8az9GHMkAxqhe&>l zvxL0E!D-l$w=}mUw-*njk6>T=C9J{e3}?&DV~SGrI;P0BiNjn=k+#c*Z7nH0tO?iyT8lNE68$fa-J+_@;}%G+kU$Ta!+nJ*SI>#c%<>#&kP<%*&moHLu}y8ARbmls=svNoNNkI-Aa>&tfi`ipjt+gHp-q&SBA@)2b^p6$n+} zP4vwKCl+O}j&w_3YJ>H#^|+NCwC=L*x1O_Jv?`F*WF-g2aMn!Q?bGo|Ra>MMdWseF zOH%9HS+aHKb7|*}>0|l~<3swE4^S`A1xNf8U7VV4wozR6U8`EZfkKyUs*34&+R6>2g)D zEw4GmVI`|osWdt*tjQS*Yg5}ad8kbLidF}$`=XeZBh+Y+LTg2t+Co&KtwGDR_ae2t z^F)L7I?79{POsrGQe!33koVo>rLnY=M6_s;hSO@)YKFl|fYo*6*|?aIT~wph+vyjC zoIX9>X3{ihI_YNUN>R%(by|+RD`mOom`a$a?h=i&;tWDhFo`vGyPbTz0@%W(=$t<`Y4EM()-6jpVnCRL})M@5`dQKl}^ICYCrtzxCR z1+8XQE7tI>>M3+PH>J2;eY<8#cQ4w_-K*HG-mSS;_ZE7KdqeSt`Yp{Hy5FPUa~BjB z)W6qU(ESPhiMyoul)t3@ljf4H?8@CiYI%21s+#N$B6sn9KuYpsVafY|7H$GsCvO63 z7pur7pfB6lqm(xGXeta128QhJQMuKa951M3wjeD85wX~CQoxz|mrrAxG~FN@d*ppR z8HqD_|9AU(;#+sNx)ABHPijSd9f!%|YnfAY!~55ywa{K~V~c;e?cHTG2O-=94Rp0S3RCE5_bm6C<`SpoLE0Hh25IGQ&cH00?70zBb>5#2;B<&(n%Dae|Em38)nmj|% zs=S^0!}{}jR?m`TrdmsqEf=Mi71EKH-ud>sm3~l{xi0Z(-NKDTuC4LCL=uD($6zNm z6@eoY0i}m6bQcl%rfKWeAlXV3tj5%?m{?VZcg7Cj4QKueuZ!5?{I7ES!wQVh~3C_gI!Z_%}Gm<0;dq+GjzrpRhr&d!;_1WToh@L#z>#zF0WB2 zm7HE{P?{K&qU2H-a$*X3*gpu9z*&48Izpqq!cc^S)XLPsRF*shrL!bQ4ShCh$zD{G zN_NB6v0_GM%}%-x&MB%HddFaf+*xoP60?`q$Q!b(FUw=BOWH0;`a|j12hdk=o$2EX z8nTscEvu$`LR8Xg^e3_cHP*KxoXMK zS2CeV8JK_noH@O5=!QJ=JNnRor2jlv`}_mw--Oqlw7kY)m?O@Ki%sSl4TcMkRT}0K z{Hcbi)vng8*7f6l#)o~(PK)YY?wsx;?!1oERB+$tpJ&2Ig*3Pq=GxFp;R@aN=`e3J znULmw?l4R<1xGn}4|bhYYpP8~d75sd8<)sD-Dolj#!91T+-X!AZJ_Wms}M4C#7HN1*glWx!&kQZgpbxU8?Gca~*v~^mO4uYQc zbpxn+30_*OE4GK7h~0MoWHz4pC;VEVg2l}5UWt`H%6}r+ zqds;c^S*-o#^1Dj7n8OJfAsmia&(Nk@*0gbXC}~ozNLwC5Nzp{NUYV6}L&p zP}hTA$nGK2#~+eCQ5#KUhnRF0(?O!O3q-H7j}dwbeN zb(aFuN44~W?yMZ?!)|j5jKMHkm@#6=dp1ADFOqG~+`pd2nJDp?nNAML<{?bFvJ`W* zIr47oM*gl(cTjtjfj!=Wa>~$gF>@PpFY^y7wn&B3Q;O2dz$AH<29f^8!GUM>DdhJO zm|UV-O`bV7FV9e^$8-Xw0ki1G$_4s}^}tD9knD3(DbNS32Tm3Xl5xb;qN3tMh*us$ zsIqLZj42zd_yMsDoxjunl6j>6*EA0O2pmkIjDNqiqzY^W|E13q${Cp2(*sk|)6&wJ zw_4XO(W>&lax++G^C~-5DN0-G*5*~1wF;c+-U@ zoz+@wuG1~G>XtHfYNkY2U}XxJ5;Y^Aa-CAC5jBjaHpK-4+Cee5k{5Y~uQj;h=R<4B zSqhBCmWEMgD)C(Dh{BEaWE3$ZM=Lj`WUZ>17J66B~E_sNpu8 zvUu48E#*J#uiA9$q5f|%JZAT1`kUKg|6bXfQPxQJ81u$kES1kXu9>o00SO;n}q0c`1><$x3n+>kwj| zM6>MAQGg2}3xpOv2Ms7*K{w&|&@GU+8A2nQgR;4&Q6Jz>!hHzh2bj9K%YeTg!Vw77 z5Y|C(K)4yg208RVa0AV25S|1cn+P8o{|4R0{{r0rWg!R;LGVEMu3|lU5b_>Y)}bMi z4rP4{c;o~82&8{c`6N03@IlBQAh0mb^cIsV&-hcpE6Wk^qh^1nms z46y^s4{@Qn4*@O#-mQ>-FT{Ni_saFnpzv=94VBcrb#PnFvgm6&W|Wzk?UiU_Y?I86BU-J1SUjLPEc$dSnp&hpUEecWX<^3gP%23Xgh0LPd*$Y9BnY5 zVBj%Drb0_ur#7bWRn=+07nDhHBwqW}^Rm#@Q=diNa!u=F=B2Qte558Mvc%twp7M`7 zqdC$%W4{R9+uu+4+vzph2tU58I$v9jyfE|0R+VqcZ-EU4@5D1raGbn0=Bk9Lpk+}N zQ*!K4QN5+rXpD>}Y{*r&GPX4tT^%UBlxkPtm3gYIf8iT?MViaIAR@ipAyf3>KlqvU z6eTVovwp1SB{Shegm7;OIZW@$rWdCNUYrbqX*4BKs2Ip$V5mG&1!73wI9!Fa*=mq1 zMm$D{soc#yOwemqmlQHwwcb_Ty7L}&w{Ja6FA6J3{eXCEpWc*h+#4VBI?D5wU$62Uk|m}ZF_5YoWh zaK<>6!@R;qAyrpW$Mfl_W)WT?ncS^5BU*1V;fy&zDY~NeO)!unUe7(Or>emaZRwq$ zH};)Wb-EK&@;OQO+XvFuBOPDTH%a=aK)Z0pe=Ah2>;7Jn(7UFo=yr^P^*gPApeQ31 zi}*~D+>4{W>lV|)DbKc0X^3&W^Gp)}*Egf-6{`ntm{dY3#VW2Q0b_{q7(Xx19&lJy zCle?TRKTXNC~>UNA`+OxTi`6ujyhdlfG|QI7 zyW*>Jx$^1|2sV&F6+nA#**HD=7?z2)3q5NX`LV0to=al3z8RyL+$U;83@s@Bp1?@<*d?nWc zRkDH_zb?tpW&#ucjhRQV(pNK&y@G#6C|d9qU0l^h8$K4luv3Z`-etG32Z}7l1XVX& z=siKZnj1Yv2)@s%X;I*hp-!_9h~S&Zs4l37xI8pkwFb@qF_nj!T` ze*9=xCV2vi5soF58VzgABse2%Q24m=OC&R?JZvt^kG*h3b4i$OecCG}NdS%egjYW_ z+rhj>*L`mdL`&*>Ao?oGD$JOKHzja16Y*krkFpBoS#K0Ty-7jzCDh^}A6Z)!@= z8$=9nH0aHN&C$t^O#CAmcVM1V=tdq%UrjDf@*HeWOCPKXRBo+o*L1!~*OA@*8nzoe zVd2Y5w`RE^mRk)hsOx-~krzdNSSLXsd`mRQ0obJB>lLUYOtrY|#h5YbjqY{`*XRXU zTmdXecS1P(R_+9Ko}X>srSMGAo^RX83XAX>UsIgIqreRYxCqS$(iHs=-AE0` zPxe^5Fur4P_2^9FOY(d`CJ^v$)96fwxqIhj-LtFc2#McxK2k7!<_8jIt<2lyH6&CD zzDJEM4>V}5w!HX zE{W=X7iA?*!LGVd7_sfO!^^J_S|wgViV)}DcJuhC%xKF}RemYTN0lq(CX-+}3veJL z-0D%UpGMkY(SUGX>+*_7KLKAakH;_OqwEam!3}Y17-Df$z(pMQHzuAf%2LGqsqdNd#$2`~fcDfd?Dngi>Snl5~K-!|u zreEs7HMcwXAA*jMQJEslY1fu95m|DzrzJ_$C=w?Gnj{=%Lv1<&l*t_yq$}mMTH=~$ zX4k+|`JzaN4-zXQ6zwx^zt@Nox1SU=5H%On(6kogs3f-dxcy-2w^ImlN5Y#HY*#7q zSmi2UTN)WS1(@bhofzg>3C851~VI( zbPaHPhw^f9@1kaY_WOinAm2ZckAOWqHcg(jI)24LYH265wn=l&*PJ|Aa0&8Vr_5p6 z%s3=vKFd6NAFiT5@0?@%3j6Y9#m`@q;FP))3?jr#GQeuWMOR97Ck5jk!8&lZ{=FJX z@As~FaQrOSJ`B{(@K4b<0lfrklU24;O!QqO;6a1{Xu=Nzn?;x@6PFR>Tj)4s;`M9* znO)sI^jU}Ah44_TAf#)%uk&cMvq2vhQMhDrPcR!Z&MjE5cU?H5_2LdTLPDz_@5Ce3 z$D_C=>ZoZbf2@D5W5-r@R=i5vIyNlg^78da0ZLvt(w)SjtWXpH-c7VkgS-2;uh;GJ zvNVZ2E-M(1LPv;3!1$D0QIRZ}!%NUlWWgY7modI`s03)OOq)rz(;G-0JSr=3Rn5jS zDmgkrW=zQN7ye1P58)}$&w`$l;boEyeu3-UbM<~uOB~KWp}9)$CCZwaXQKxV&_75{ni0f z*i(-`f&IC=&B5mQV-O^u+#3NmpdPZv!Cq!RRxdEsig=>(sV-Rog9swsj*K`!mAF?( z3{d0A&fHb9d6ThZ4WoyB1YLkzrhV*dfsD~Rw{7CQ_bXdIM_{$Nf}Ut{P{%tN5^gRl z|J-UD_Nunr9xgfFy_HiK7qOq2c3zkB&~_@o$!WW+vatIFR)|EX#1<9nCYAGxo|O7^ z$l*GOEF1d}DZyrKD{VWIfZ`T(R`c8uFgT6M3!(w2Db_D`e$Z|Fj^>kJQZXq}Mg1nl zyWIl^xw|z4jB@oSQB_ffz&-%X<50OM2xqF>p7;1kzKTQ0&`&vgfDeo-1Q{#fFqi!M z3v7trD2S0T`KWHYnR_bzb)v-D({-d=ny50#*8zj#cCoDqBj=)K_!D>!K3Nim$l3zM z)&WQ+#6(Dm&>FAA9dbzS#eyJl1hOXKtg5;^%-^uAgGolTL6DTaHwoz+=DU0r)#pEN zXi%}ZP{@A*M+;3!3kO~`I)p{1VafcEWijat>9>RMlIQnWFKpP)b#FXj$N9GN;H;Z> zw=)SpSgoglfZy;<8S8lSW5D)#M8=^bT=`1~FMJbEz9BJ-hav=Yr-)y`M5ve@>J$P^ zug^2X@MO@mW3R6L7Br8VoLg(|s4s5*X4B%nxvoZKVREK5v$Q}BB-+@T=gfa-fw+qM zMNq?BOrdykbz^RAb%xl%38ws{WmzMQu)IF6y1lZvUcbHy`jC@T563JZ5mL&X3rs$H ztfamo9loa)EV(6s*4f$GTwkZPRpapZyX?OIJl#OV@KJZ1?Va`_W3*#>}kgb#%9JQLx~(M*ZuA&aI;Ju=zdr;NvX}uh_cd zz2>0vU=(lIZ9(Tz(`(EAEnwmfomb_4e?jN@>(3+Ph4DLdz6G6EO|Pq)kEO7zzBt4U zmaG(U4g);#=X6VlMAX2Btnioq)j5Z8WOZ*;8FPV7zn9;KNShz<(`dXhYRTxopa_qa z!ahoH4(t~cC@{q&)Pyb?rpP&9ZqWA~rq;iTyNeco{yi7y#q4P~^;$liZHY#!^#@@m}f1z4E!tjJG}_f3qKm{9xi&!;~jG5Z0ZJPmK7W(Jh??Z#et6 zr;s`)SG4Wsp7krmBQ7}q{q#fpDW&&S&e>5s@{eRv3W%kptSUayJHUsFXo~*E-h8c0 z-VOl6z_%1g4+|tr`2Gk}TZeR9(4u;pJ$1i?lgAm&dvA`p{j^o7b-YTZo2mdSHY@Z# zX7`s-Um+EiN@etRXg_mgNxsFpx}o!mcNqwXUpV6|)Rre?5Mdjn02HblbUTb5Jjf`X zew47rY1dXz*U(`@4WBTE*^|&Mr$0xcMF#q&8%~dy9W+G!b9cD9Sm+0E!N?CUAVtxp zRDx4q8=RgK0Ll=;LN8#Spb7}rr;RvRFRE%ID)p4CG0+3&%)hHDC>k+NpJ%)V!dOa? zEouxyp+vYb!~@_cpeyUI6*MlW>wN}Hz{T%b>th*~NK;5wRf7`yd4OKBn2=)?e=nsRt zBJdxmqhh6}JvU|I37zhu<)n8jxGV+A?Op4=*F(^%tw7XkRzH2_ybW0tW7je3JIp#Y z2Jd~7+q2QQW)b}o$tm(eDCq6dmW)`j3k24USRo1oS9@r4uaHk4a*S(gH_~hkRwj5S zc=?MvgpviXs7V*pIr7*PGaO5o5_a`v?s@6EJnAmrm?908hR$@uWJy869omqyGiHw- zK2EE5Aak#DT{lw~YgL{Y$mspd}r0CV>6 zkxr8O;&0kVLRYnv4Vjya<0QvChiAL8mC0U%tG8d7VlVt{tTLG`xH)BEE3=O;X-8-1 zYD3~`Rc$|gmLgdv_OXpC?TLmXFX4%W6Bf|RJ<;Y*<+EvNlsN0_8paWUMc|2iE=Ir$ zXUozzA!4snKfzeb1Q1xmk2szQt{Rh+Qwe}wgfS%Gkp%y0s&p>6Z#*U=$6EC$77Ro1 zPHB&}C-6$thY83s>^W2l&hpVV2kRr>do+ec9czml5eF9C#HlW^iG#U7H& zk*ql?El$w$>17JC^1s>6>iNy;Gev^5Ub%j50>Wq9GKpNK2I@VX7x<@@8c|074049}1nhI*7LszxwJmTVSU46HGv8G$pp7q(8v2w&4Y zrpx<*A3*qdym0e!@C)OR@a~UE+NSRcCV&cHXu_scq-J!=*{)STPD1)FeF*;yfRn%@ z=k}%>gg+PIPt=|T6A6qS5`W4=X%K!ypim%LPzn(sivMF5Ku%7R69Pg7vN1sa68WF# zX9mc>ggB+R2ZYa$Xn6-<5p>zVM8P3QC~fv%LShOGMT>t*gi<&g;3EhM9U~viPY4nK zi9%HHZ)HUg5mMQ|L_rZ~s6_T(LP8D-C4pb=8zpEqz*`Uq+D9IepAaYj2nD3z-^$6} zJ)kIL+4OSI$g=;(me9++Px0rc82{4mn$3=YazOcS+Gm@);IHzhW^(l9Ie3>O;D8`G z@ZQBC{Wl$*p7+1Y(dls_2nvSuo2*;}r~XwiAUQlRb;dsTUnesQ>Wqusr3DG<5VM1U zL_jHs;)y^|{S>=zs+|M|6w{quQT{Y zg<+ne)s#HIH3h-E?z5hMwJ-^V<%Nn{aFuaLi?!inX38Ca!wl}p3{Tt&Ep}Ef>Wt+8 zoB?1JxcppQK2ovT{#^=(j*u6E$@+=GuIVusy9hUfPw1}ww(r~%7zr68y(2p#=n0`a zBR3MFHpUEY%Ta<0zURLdGHl);L12k&EVE_5D16zQ=jUq$eV?Oe$$b*q%0qf}gYv8f z#i#3jNYZUgRg2(Qo`EV#^aHxV57Y8@4=Z7oI$5nEW82f2Fm_fz@t21S64-EAO;_*p`VQUte`{H zS?w=V14XqsxoLUhI*V&=t3jtYgl7pmc+qKnYjQ7c{H+XTbPaJcTS|DJkf>sRY zV(0|4@x{>jvDbO`sDM3@Ge=O3fb_mg)sPXBI~0KU*f@UxFK5IH$5*p6{AK(qFQDCz zFDNkQSBdna0=I5kU$yDXPK`(KAPgFLj^_bCG4Q_++4}cQEJmT7SMx!)*=$k15Z~Lq zco^1FiNCYXx+BmKnRQAg8)hOYoVK1O+}kXHxes;BpI1Atu%QXM&3cBt!4{KYb-j%# z{5sX@szS}D2wzq~AD^abSZrJoX`Rs~PbYj9871kcH#80gmrYP~qdaWPw@G|-(K-9d z8bqX5)ETPNeQ@LYO2iw~Sn`!BZX|oUn3*a;qIg>|9J;t{w+RLPX=J%1e-MF! z!)}kl;>S=z3@?FkRj7cu97%ws{<53$R>RPe-YhGLFzwy$#E;(*Y$7O6r8a%jfy-_x zB8kn*Mo-P>;^n0@JWPMCYxZ!)bB zV6(&PEa5QW5)<}Y5(`T;!S~FkFKmoR{5WEQmffC-W%gl(OCaf!m?qJl+@fqGKTHRP z4%5=UGl3()-Dd5BgjYAG<=gud0`s@>Pj#iic?YJGfH2QdkpP$vWt0(_t77~Qfb6

7DwpMVcZffG#m|>=$k=?k^mYvTOR(rcs(NmU5zKg5D zT2&c#iaQ0DPfUy7BQsT9o^6R&f(!uT5SA&*pBzCD#;%JySd}2hFs^jex z(>>;i=mJ5PfkcHpFJRHj41g}~pp*#p;dM$%>M%9QNz$cMKN4l&9#u$Q!zz_svoqMf^=`4KzBrwnuhHSaXxTWJ{{KM`|1-ls#$N>Kf6fs9 zgX894W1*$z_@rt9slY5pK;%R2mBAy`fqQ4di+6V|K0Mpi@)6de@_wr z6P~}H>7R;!57yss@PBXmyXMa<{><+mgYl>GPscyU|1tg^_HX^C&)*e)PQ!on{HObW zHJJXO?Eij}|8?{~gZx9R|J41v@yQ$ipCrxx z@P|(b3)G!!DOwwAt8lFfTrVuREido0R?fXf$5BBC#V0F4@3w5rhErBfYe&~k@8|a? z)hT;>FG)Ma8yj#dLbaL=wHvN3+|qPsL8Ui0N6+*)2r?%-gGTrE%eOODF0UI)Z#V3` zcQ2mo^*2WKW^8Y)AMY=2uhr`hXQyicjXQjF`Wv+47XcmTRYy@fEhoS3E*IHGpMK8K z?l{fe_U^B#sr}r~2(o-|=F#%xnlMh#D$!P?Q8J%Xu2;cT@~U{2oKsQqDt~S{PAlnD z@_3mqxK0|;q4Fwz{T~dNXY_-A4KMx|dHlEhA^Ky0r;pCRtd3M36FdWS{#CVc$o3fF z>G!XzBj?8q&!CNeRq-7nJo>$R|Ep>=u*m}C6xyO`0dAH@MuGO%4Hh>YURnCuWT(~0 z75z3E?+1|BbNbPL^Cg7iUpUbjI9QoD{v9#S#K^(?AJ6=;b|@D`k$HMeN9&FCMv}Fo z@6L(eIboKP$e@4)IbjG-mx7_posgQNll`?o6+w4Ef{BQ}!1^6he916k2*$>;)tnTN z8*mPk6Gg-g02adpoqqbRbO#5V@$iv(;q_4vXFpNgQlDvmoUY(#O%6%y?_h$uEq62< zIvYM*prIX@-U%|xI#y|}VyV301Hgj=tX36`UcKc%BoX&15s{#!)PT!m))}4S`JPG*NZpk#q6t zA~)|BuQro4u%a|#(D}kt`Jz}RIzhKX-r*ELV7UYaf<`FeUxz{kqv4k1D4>P5C_qF} zLvo{$Me;CE!nP;rQKFD!i6jJvDZmPHnd2iFqha90QYc^y^AIT@B82lMP2$4_vbmG( zd8zmViXbCI`b}`C1a_uCnGvN0s_`3$K|lE&K6yOPuz^5w?S$3BOD-+|suA@3F(m2!4FQkCr76 zzsR-T>$>xC*@D>M*pO;A784W~hDL(tk< zO=5V#)a+3Jt}CE__h4;tY>|vnXO7#vd~{cleL>jICfn@I03tGj7-L@uRwr~szs!T7 z5tN)_tivhihnf7*^^om8<#7mH2S1$=-FsdkTuSIn;|AdI#}kdp7MpS#YDRx-Mw8&( zop1$Qx~V7DRfDS&va|%vE(Npt2sdQ^npD1JUIscEBsj3W1>NQ4NbZDt3ruWd@mlfH z@Z?$su6d(b4si18%kp#VoXB!!IsmIfKNI7L?hM;M70T+GP}>z+n1Z{NalpJ4j`&gy zS4WC2Goaw@P7^Q5X3161S-8Aeo zc=*`ZMKRRG+Kn^xxU}dccC+~E$-!m!kkb9q$@-z?XXuGgdq+UW=?$OTdnl6p#GBlT zE+6FWiDvZS&L{Q`%BH;X^zen|2R=^-_fmD8CQc8K+ilL5hz4xExMq?2bo>3rxu z^g-U@Pn$KI31=6cSBJX!A1rfne%N?3|U~<3NL1H=JmtpXQiRd5LKDtLoEmq-oaFM@FP;ex%h? z3|JB0zuSd7a&nW0rblXXx0ijU?hQ?6fu8`AkSVPDnn>ml9UShD6NE8IIz&YR+ zieB#!RZio3@5X#AXsj2<$pLj&#KLdpeGO~h@UW?;PfyyUr!RKz-MSEC#q)D)u|-tI zRPAn&$lbsI?zd1t9Zgi*WL~xvp!$A@J+DN>!>V}4j(Kk4F3Jf?wRK(`> zwL(HVXLwcUyHCsK{M4e6%1x-sDO-*^ws=TR2P*?IU=M~>r!Fx?^LI;y7Z0on}Safo?dg-<{z1&0|#-URSR;%aDZsa`^sGq(nPvv5*e zA?7*9($$HmZ7B{EA2z-1^8#KInZ`4vNnWumDCjZR55*|RORc?O5mt}hq6iGD)n#6# z_iZwX6|d8+%fniUQG|87N+Ou0Ipg<@qWntO=ptWV)rY5sJk0y?JNopU0g3@{FS;Q- znt`2ce!>d}i&`2S78?k2%#w25;0`Qz6 zmndj;*(VKer2aa1H6PhxV2ASYZ{LQ%^=nE76yn4=%dN_Ds22|fNXKKcU(2Kk6CL8! zXK)Y4viBjCJ2{PL1gJ2uONxv*oWuo7ODz4A_%*XxP?1)|?z>npa4zRv zYF1%xC+aJ61%T;mk{O`5p~E(~ICpY#KWw21?3w)e^@;P+=SBfX`7Wc~YPiQq4q(H0XCw2M8xFyOwUy36KlIFw3G6FclH* zSijJgad=AI9+%h>pFmXsC0NR#L?F!HuYH+np~-0+w~aJ0sLIpIdIo!QnexSuS~C7k z)Kk4`qFvIAuA02<>NN`7%{+LEC*b8ZpR_uJ&vlHBP|laBST6~pKSvPs4ET6H4(UFT z2zo9|$&3!!V><0xG*D$^RD?t4In8&;=SQT#<=Inm#5pDWisaD zF>m|HO+bEks_|>Q@Ym9sQJz&8q@yLFObDy?dp_h{Pz?Agm2EQiC~MecC=1=xy}&kv zYY0AxAXkX#5zFTfqTasWCOA;X*;PdMFHy#i#vBa&?b(SJrLObjo5r^E*;C#JKs#D= z27cS4jM2ofjBv4%NTnV)4K(u3#h;%C>DWa|YM)YKZT#R;3GdX@&F(EBPt$C3-XwAc+^2Ha5U_16j6!Zr5El0=v@L}QMZLXd)XvFoP`E)+CjS#J&3Ig`21oR(pF zKUrr|%fTtFYX6oh<@rWy@6e*vVki^EIupUR?}D%)KA6wnY$3(?N<-rETCM@7UW9~# znjFNCF~o%k2o}J}iq*p~^ zehh{P&pG#7!VU4p%s8;Byk$oO?0J9fML+V-AgI-ol>p-xC&Fk=Yz|}8?|Clqfp(K@ z{+@k>)|A~{T%KLSOkKYFbJD0;g#_W_aD&N!Yfr0ztJ^VpyAR0FVbIYhY60(bR5jHIGDBF6Vbo@5|DA zUNmmP63;TQpHvN&3d`EajlxGdf9Z`~f=xK4C_9X&WHdT8TR!e?^&GK*u=K_xv5xSWQEcppbCW6S^fuGZdztYTPb5Cb$UOiG; zY8OcO38oS_TBPSa5ukt$xjCz-%T$fzDY@ljaNyd1y1Q$u&Csm5cNBflYm zEzW#TIt`;oPo)&8;_J=(@@V?x2ecTL%CD{7gLj4E9_#{wEsxwC6%~ZUsLy8nIzkvuN+pyGQ+@zIg+dtM;0z8p^oWl^Co+_hTR5&*WU0?9R&{+7T1JoJ=UFF@9z1=KeaI$xcZg z>;B#$^J^`ckCU$S+^6PSm~jM56vm5*+;B~5c>MId9P&j=-q4o2<5X?O17qn;uGv*y z;+8&{O&Zzv+Jf+q@l=)`Q)zq__%>YL0rw`J*$onUOv2YSz04%2u|awSq1E+l`>R$o zE1AW0*?l>2u5TSFKih4Pjwb1+BUOIwx?TDBnf3ov1Y-obc%rOkhcLl%kx&-^_P0bs z(S@$F;d{|Mxe8hd0}^v%_|bT zu|D$4hj<~#4IYIaWB$R7pC#UM)74VQrt?q^`2JjY_d()$Q)&J%#uMo574Z>zA&POf z&-ldVgAZ(ic(g;n2!09RRcf|W50IdoG5g{-3*IieSrr(Ey&1HC#LFNuI!q7>&gXAV zyHL!#vlkw&Fc?w@wyr!{Zn)Z3PlQ=bL^gy;(%0W-0Miy0RSo%CO|;=U`mDRL#C>|5 zff05#e@D2eed?Y?r}p}}-$C3})ln6KX5fi~t5!@V3ZEC+l zG%C}^nL0`Yv}E-%#vgM|WpxYx+`iT@h&L+`EIdc&2wsn7nKR^yNfy|{#oUAq9t-y; z6deKa2zz)&+ktr4h9FEf>DFl2B>#CMo`vo)VNY%g3@t#MLYu;V4v^175bEkCy&ux6 z)+Wf=wZ*FP&qnA6a5V3r?ipl|PG3$UaR(B2=pD-2X*QpuU$d)#xWv|QC81aP0iR-XR3c*H|6%>U)MH&pFC16A?MzY(hYV-JbDGaUGsa= zE;~hJkc!39O2+1E{cx>-_G(JIMyT{zyd=R3@Rdah>j8}v18<6(&7%!+)or12n=L{o zcNH1(y^2e;iULB;vx~EA*j=SKF!2@Jk!y(8G)9pW=CD_=kkZcha`n36fJ-*xA`n53 z5cvzlo`qr};lS6Yl`(rz=$)H-`1S?n{p|gkXO&m)_eAR+Q%=CKPgoVYcwn_yuHr3F zLMKZJI^VX^V2=wvhpzlDwa6b_p&|_Iv$(nujB5H@V&ZKkCXlTZ%!EerDP~xNML>RT z$7Q1~iQ(Dg^V4D75o-EwIW=nWRM<##p@HCpr@4@Dswuw>Xh`J8U#Bhmw`?*W%P{Ce z*#pS$C*5WU%G0sTsqT3rTMI5G+F2@|#ArNQQZ8o;G+K z6`Wu0EjyqeF*mx41cNF~T^uuOH`nzMM8ctZWtyq%^PtbUz`3!&w{b*mC8{yrVCv2`7q7(=F^Ja-uC$Wf0RS>B@W zAvs=EvzLJ0w6|ho`V<>zrm~C+R*fv`_&tTE?R_dWq*S?rnoDNC+I^GA3u`|nAk*Lm zaxkjdu^2OP)IfU+xkRq=%l2Ff(v2Lt-74*JK!r_%iRJbK5S4<9pML^@MfqV z94Yb*F-yoI1=M6hEXB3MjSV>hw}F2@BX&#aevJ%k!7B1m!$QH7u;HKJ$R$tS}V z!VM=jJj!x^!QYOp^f))a+1qTi&A+h6c|y@~)`xJ;_re*PhpBl33}A(DCR+$ymGz=w zznbtDiLJE|NGo3UPfk+27@D`Ypa|+KLyH|dfvt_Kn)z`*g=;3+La^Mks%JYWqE1PFSc9QT zm*2_L#~7Qk6i#kc8_tMsA(S_yZg3(mkVKhpiUT#pGoo|-(!8B{AJZ@uA=B1sbk_^> z;1nS@a-CUh>5@yhW3!md0amR*h87?iWllMg!o&)1oz>x?x3zfpaVNYp>fVA~(->~D zE{lvG$8C2W*fQl9!0-$Z+dSs0Bk7MHGxRheEE;RYPuqt&lKZqrP~+KwinXR+luN9zpU0T(866d zx0J#H8?CYyZWTu_6JW$xzXyNV^lP0__uLpAY}6A>auycUnJ7oHRG2t9?Vncn+I;Ij zK8~*|5!p#R#ZKzfw_49lSqdq@?Z@R33qn;{8)lhZl~Y1hp|=xOH8TE~VdxhTnTmM| zAnlDEEL8K08?$Kjw$?v^-)zB_r ztF|7Te}<;;TKkJJ_!#;7@?G&p*fWqbe~)dg={dSe3Lwi3t(B@~!aUup%z+l;M3ko*(J@S} zuU{J*2#k*Vh}Ty(?#Oj4L$b4&tSqmYGCS?&?85i2*i% zHfEi%2H}NweCKZL+Hrz=4d2qWFm5_|IFp=TfaT^ zaC}1S7|t>M9ygqVW+oHHnvy&7=cJUu`bd6CR$p<&_a=y4Kp(#Z{V5dHZX zQHsWzi^Ti|_s30T*F&BrO8Bp4mT_BTi5lZEj3`NYciV!MVyE$R`Y%xjrDW|2it;3i z?Ob*)_d`UFPz)|eR6;^-I7)yJoQc-W#`oPi<}SlFAURSph%sY6E!))u0l1c{nmdJk z3==<*xOK#w{urf;Q2970@F>U#PC++eG1LgfYor3FZw?8Nv9e6;?2~0d88Q}R!x069{GwH)8i?4ErBy=h7TTrT4mtai;p~#P@u(~! z)CUW@92=Lj7UdKhJRY0m@Z?hM=5O})Xcrel10f2_p@mYR<#OW9sM-1`?%Uu51~Ixmzq(yp#b_4TiKT$bV% ze;$Kv1*(s$sN`p%Qy2y%4N}73K(R|uUkbZBmyJA7>Q%hHTZM<`Wenif0T^J6X8Q<2 z(#wB2q=Qq_gQ4$9fWGb_ET&ER?MCF6zN5!SW-jr2O7u4oGu4NP`MgG&A0&Ii0C~*q z5u_8?Rhz{=Hq&!wvMZ>&SGwyyEc}fJ<)Za#BG)osVn5Z&mL2Los&`L!Rp_jce;;0d zs~!jV_F9mGUA;Oyj9nuafna7f3#G3I+`2GYTogg zBx-v-^f^^w*^w=lk_3sM@5I2KCoexQZyHn}XYPKCb{D&qAsqH|R<;27))G*l`QGtp z#?;xTiR^$7^MsvD#Zf^Bx2MjsQiLY<0f=*4}XLnAy{Kf9(qY+|=A4mGvY%8jP@BvC2uygD=x3(s^|^E6RR9*dNt? zkx;;!MK=z`$$=kF0Bi6|qGKS?fx;2Q?Knrd?E(^Us*hqVpnc-OF4&_=VuPct~rgcVlziyg^Q zd7l&Or+dEJ=iD)DqsECgD<~Rq*roV-)LO3_ZD3#w zLSYQ@Vgx};=Nf0UNpugpU)ft_2>yQ40-!vmGwKLrE`AegdQGg3;tgPeT-^|%$mu%} zRuANcNN_SQ(^<^EdU@vCx_#rr*&iP{zft0w`arr3lmmWjz$ZKp7aJ}1Od*oHlxu_8 zrXxMKFC{`A(*XtR4;*;-anfBZi+7J6C{zvB;^mKq=m~&>xkL!Yxq+BQEcP4I3C_nN zx`Pj31G)>|g&xc6wLzDFRUKP;E?s*cPsAZWqo$6c% zMoQX`?ZLN|cg3_n_T6psHqLOy%ekV}ZO~<@sHqRiT<1*l`uMG%F+e@i+3MID7MiV? zI;*+nr11++Em7`bi`mTzB3wzX9r-V@)^=ZLYSKHGq?rFW}PYe+ewQCk|koS1NBWB#-+Z#US z5fz%J#5^)>47RO5FnsU0`tm)oe5vh8bTW}<@?y&e|M6+E5>wi!6tH8wR>B|0%6{0! zp(7+*&hnG^0UMEMd1n@`(*&NAF7>89zb_0>iP3?$=SSw(A#F3d#IH;i9Fvtb)mBeh zVe!MqaV!pbpY9s7E7MttcS$=NsskJq(F?}(yubxa9m9|t zfJFNz#!e3)?TqX)5J>{;I?yr5X#2s5=ZK&tIp4QuXcIcz{j_fzO5?y-G6wnkUy}kd%-d)B<7A*;aHZ!G}nVFfn%vfe-W@ct)W~g%6E;F-TW@ct) zPV>~;(Z1>4+3wvpnm;4;uQK3{=Re z{cgi7iQ5m}W8$W~Y(0u2BKUAC#mH@AV<3s-wD!HEfH=1f_Nz_coC(*JMRAhOlZt5g zeu`Klk%Di8Vtm75{(w+tD~GOu|M+tyCB^T6O%UT4|EecCz<1f>;Low4v%JsK&NVDD z*8IIX-(21OVu&V9CC8v2S>&8Y2col{;NOvJGLC&=ne?5po%|XKQ_%|@s?`TSRL-03aRB?p*(&55E9CH;bu@7=N z%}Mg!EP<1cj*Y6&5qgz9=9i31WeDA|?sT{$a97axlsKQJuAGdH|D?^3k2~b_ftqPB zV|)61JN3hk%wp2@YQAr}`eueqV0Y<#DbmZ-*IJe-oj_lyc30&<$V@E{XFTaDQ+1lF zTZ6z0kMzKa+)o&fcTny^nyLhuhhddfL>T(Z%3Js}p!+hxMRI$-#XbY8W!Y;vozUy( zcEG2(r&mbrHK&8>4HeSI_%c20nSbWXjVn&V#aG94vUiy}b*OU!uydXa4#Sh;bDI+Z zfuKXRa@Kvi#2aS&#>P3myjxyugi?a9f@SI(hTU_o@^+eozA&0NJaVB`spHTfM7Gw? zU=XA9P2M`FAjpz>pAPU+mXt9Dh8xrgMBzS2&C~v23!j44Ezb|!Pp_WDNw$o}yv()I zoXO;*v$O6SK$9V|#EuJa>1b>WKeFh#crsqS9yjm2sb%0}64J1#%s-CQ%TG&o9roL3 z5jQSF?&kKmt-FdU+mxbq$cM%=gkB9Rv}{bx<8vXw4uhyd5x4S#wmpNvh?8Dm>5CSW ziX;;!_$;F|(2M-AAN_KS{L?62m?|)L>*DxZY?f+HJNt%dsL%UB5Bf=tItfQuFpMa* zzJLpJi^)z&{f*B7s>1YWNZtGPB7U>_bar1mrk)Bp$9a42nDc-mE@q%(?#?)tynH0H z+u8Z*J=@o0Cih2fXVZ`dZo2C3a~q0HF1WymX`QViedsU0-)qT6`DZh~Rx*Br4k314 zEkQwK3{vDOBB7GP6XU}RR?dEj-T08*6&}uu7sL3WI4C(_qvK&^IqI7ZaRh%7eVSz^ zi4irLA_$=3-!+Ob5)}PjNza8HdvQrw>RRIQy~-V{jFQ6AV92|`SMTs$yjxQy!z9qc z;PE;;7#Y=(&0pJ@9(-&d%ug#`z*YAVBY2wlcAAAIRfarWZCRk}=`fGoCRw|IV{272 zmdnoT2h6l0NQG7>b|Bl#FjFQZjtBar^@emL9^boq1=kAsX9=wLJKvD97toK!l1vAf zNp44nbYr~B*1y%%CT>c-zxwW6b6BqTPsNPlxpwr8o>xBBYu|Ca1O-?u(Iw>d%xT6Q zzy}3n*lKLefUC`rwd3}tbpUPJV)QL_p58lM9W+JGOQn{*Z`b!e55um)I&y%`czl!| z@;J?Ju>MFX9IH2r1DdH5q-`NQI0wBa4o|sq?oIilM*wxd7Q3Lm zV9rLkUQAUjeZTot^NEC@TkA)m(`E%8(&9uMTI-VxOIN7evsUAZr!Y-tVB8JXm-ZUQ z8{G-w@Z^zTf%{heL7*`=JIt)5sFiISskI%`+Q#cP2^fI(y(HQf&-LwJ{)6+z-6PjIq!-Xr|neUbF1#|TkFI~U}NaT^^L2T_i z>05V$zk>KG7@Tn?@dOpEUGm;x* zW<`#5hYG#n4*&81Nqj_GN!kIVd-E-LW3(HFCp>K4E%3#)r&!c7V5^+M*ilB`Y`De$ zLd%2k$vH$h3b;$y5^~qNf2>w2@dD?2mVAHS;2bl-!KE9re(PF66ze%dI?&%23Z>Mj z9z4HALQpWG9eZ1+Uysu2rN9$t796bYw;u_={fz4ZIf`{MvR@b6KIigPua_4B9?jjm z-9Qd|5>Z>zwijp%HE4yAdDjhB#l)TuUAH6tWWRh{2`Y9}OPy3UzZ&{I$qe~!ODEq> zN@tPd}9ywy})`eb`)kcxmr za;b>*Z@0G7`ZjlvzCmSBS(Alz{vNl4T8Z=B-^1*S{5_tbnwf;2ZHw2Rc6Vqhh^UXL zCP-J|Wcc29Rds7`9vd2OvyGeob9{i*F^LHvQ1{~FT(wILP zWvy3ocmh47d8##@=Y6-zEd3DMcNy8VGFN={wr3v=Y?G&EC4MB7vbK|y%?d38pMFdM zh+%ae7M)P0r$bEMhr6%(znrVS z#6-;QEs&76!mcI>ziqZ5_2XoHphpRfV092eD7|ItX|en$jp+`8MebY>G~3L4Z=wi` zsX)GdnQY}SF}5Yi>}UtZT}Abg_!N`sHf?&~3m!NoHV79N5Tl_*n=Um);L~|_%J5ar zgL;5OtDHfcbC|T1%jp4bQ-q0tb%q*q61Z<7G$oJOBV(`tp(rq)hv9ymJKJF)^TgPP za7#`+4-mw+Bdl7%g>+6juP@FQ;4$9Ms}8bJzS?d>RW0McPpJgJi=Qn-_!W!00?X+= z-WD#SeqH0pM4Ag>LSSVU=UyaU?nvjcxsna4@YPEQzWlz#{rF7ab*+;}p^<3kW;-dU z#fob(ZUvSY8nK6527d~{+BTq{MtWL*)kmU{rZ!6ygg%xo`jJ}4Mbj}48FWUGXo$ZO z$z(eCSyqkQ(Gdki?Z4hXUCI1pEI7gTD0F|6jd5|t5@c$=vL5Fd$bpx0L>cYUS$_#Y+kbF9A>J0|eFY0%8(%5b`9g9|O=& zQl=6I)=EcOEm5-Bmtnes3T~A(`8hu^F(PMT8V)&rfz0U};p0ak(5>M}vbE?u{rIH+IAy^OWQB+psmk#TP_RsfE`X=wCRKmljz)uE7Fej%8P}-DjE2o%Lpq>dVE_|5{v3~j9x7^CT(b}{g-lu zq(a_{{&%ab(HYuioi3D}ukVloCan6zimj}g3k2N2SM-ZT`Vc0eLA;m{U*m6e_3r{n z9MfxrJZGug&els4&x60_W;#=hEz<%taz@$>1<0XVHJJeyUvmn!_&!|*hSy)R*J6nb z%pLV+0INTgrvmw{*aAcc+aIq)sZPEYmf{NIfB5b&xr1%@rA!V7)7W&$GP)bBSK;AV zqyMZLSFde^7BV+bT3r1(%Ou6ae0v$bu5Gok_1xFpZWaW=&$2W@()8XZv=fSFN< zUx10#qqGoCL0ayBY)(c>oLfrq8!@-{0(t-fct7Q}z4mH%$#5Y_y1s1gul2wt63IBn zP2HchGpR+aR&V?ZQ|EEG4UT&F^rV-mt5J+TPP!t<@E8wSMOgtp+XQA)8^JU+es2f2 zsuk%}c3YFxS7!(^3<=iBcp>(#5?3-Zm)uLcO2F1u7cO9az?kaKf4A80g*nLx&pi%jS%^|hC$da#incvaD^-I>Q!nYA=T~8YIJ>Fy15f|t z{qahx+#MsGfdA*v{cZB>Wsgn%jkLmGYg9@x-_>Il_wps$=clCY`SqIATD4DXv+!bg z#LFBa5XWi>6)Uoa0Llj&y1l&nrRL-rT8*~UALA>|U)EM=6>5Lno-Ex;O_ir9I0F`g zwdbgBZu;J4OUD4>Rk6Xf1+(FGeA8W$;91dNuG6uGdQPzltBMzCpK~PAUl`>1hS%bvTioy71nJ zz&CTOel6)|Hercs=OCe_;)DD9^{f}qmz9pylBYuHMbRS0)U8&mjk(U@Lcsr#qg