Show the metric's agreement over its tuning cases

apps/backend/src/rhesis/backend/app/routers/metric_tuning.py

@@ -46,6 +46,7 @@
     MetricTuningReviewCreate,
     MetricTuningRun,
 )
+from rhesis.backend.app.schemas.metric_tuning_metadata import MetricTuningRunSummary
 from rhesis.backend.app.services import metric_tuning as service
 from rhesis.backend.app.services.metric_tuning.invoke import MetricModelNotConfigured
 from rhesis.backend.app.services.metric_tuning.reviews import (
@@ -101,6 +102,25 @@ def _resolve_case_or_raise(
     return db_test
 
 
+def _run_response(
+    db: Session,
+    metric: models.Metric,
+    organization_id: str,
+    summary: MetricTuningRunSummary,
+) -> MetricTuningRun:
+    """The stored run summary plus the agreement, which is never stored.
+
+    Read here on every request rather than written when a run finishes: a review
+    recorded between runs -- or one a run has just invalidated -- has to move the
+    number straight away.
+    """
+    # Set after construction rather than passed in: the stored summary allows
+    # extra keys, so spreading it beside a keyword risks colliding with one.
+    run = MetricTuningRun(**summary.model_dump(mode="json"))
+    run.agreement = service.get_agreement(db, metric, organization_id)
+    return run
+
+
 @router.get("/{metric_id}/tuning/cases", response_model=List[MetricTuningCase])
 def read_tuning_cases(
     metric_id: UUID,
@@ -226,7 +246,7 @@ def read_tuning_run(
     organization_id, user_id = tenant_context
     metric = _resolve_metric_or_raise(db, metric_id, organization_id, user_id)
     summary = service.get_tuning_run(db, metric, organization_id)
-    return MetricTuningRun(**summary.model_dump(mode="json"))
+    return _run_response(db, metric, organization_id, summary)
 
 
 # Marked update rather than left to the POST-means-create convention: starting a
@@ -278,4 +298,4 @@ def start_tuning_run(
             status_code=503, detail="The run could not be queued. Please try again."
         ) from e
 
-    return MetricTuningRun(**summary.model_dump(mode="json"))
+    return _run_response(db, metric, organization_id, summary)

apps/backend/src/rhesis/backend/app/schemas/metric_tuning.py

@@ -140,6 +140,34 @@ class MetricTuningCase(MetricTuningCaseBase):
     model_config = ConfigDict(from_attributes=True)
 
 
+class TuningAgreement(BaseModel):
+    """How much of what the metric said the reviewer accepted.
+
+    This is the one number an author watches while editing an evaluation prompt:
+    change the wording, run again, see whether it went up.
+
+    ``ratio`` is ``None`` when nothing has been judged, never ``1.0`` -- a set
+    nobody has looked at has no agreement rather than a perfect one. It is
+    computed from the stored reviews on every read and never written down, so a
+    review a run has just invalidated stops counting immediately.
+
+    ``judged`` travels with the ratio because a ratio without its denominator is
+    not a measurement: three out of three should not read like a solved problem.
+    """
+
+    # accepted / (accepted + rejected). None when nothing has been judged.
+    ratio: Optional[float] = None
+    # The denominator -- accepted plus rejected, and nothing else.
+    judged: int = 0
+    accepted: int = 0
+    rejected: int = 0
+    # Left out of the ratio and reported beside it, never counted as accepted.
+    unreviewed: int = 0
+    # The metric call failed. Left out too, and kept apart from the verdicts so a
+    # flaky provider never reads as a bad metric.
+    errored: int = 0
+
+
 class MetricTuningRun(BaseModel):
     """The state of a metric's latest tuning run.
 
@@ -164,6 +192,11 @@ class MetricTuningRun(BaseModel):
     errored_cases: int = 0
     # Why the run as a whole failed. A single case failing does not fail a run.
     error: Optional[str] = None
+    # Not part of the run at all -- reviews are written between runs and change
+    # this without one. It travels with the run because that is the one thing the
+    # tab already re-reads while a run is going, and the number has to move as the
+    # cases land. Derived on every read; see ``TuningAgreement``.
+    agreement: TuningAgreement = TuningAgreement()
 
 
 __all__ = [
@@ -174,6 +207,7 @@ class MetricTuningRun(BaseModel):
     "MetricTuningReview",
     "MetricTuningReviewCreate",
     "MetricTuningRun",
+    "TuningAgreement",
     "ReviewDecision",
     "TuningCaseOutcome",
     "UnreviewedReason",

apps/backend/src/rhesis/backend/app/services/metric_tuning/__init__.py

@@ -13,6 +13,10 @@
 Package façade -- callers import from here, not from the submodules.
 """
 
+from rhesis.backend.app.services.metric_tuning.agreement import (
+    agreement_over,
+    get_agreement,
+)
 from rhesis.backend.app.services.metric_tuning.cases import (
     create_tuning_case,
     delete_tuning_case,
@@ -62,11 +66,13 @@
     "ReviewCommentRequired",
     "TuningRunInFlight",
     "accept_remaining",
+    "agreement_over",
     "case_outcome",
     "create_tuning_case",
     "delete_tuning_case",
     "execute_tuning_run",
     "fail_tuning_run",
+    "get_agreement",
     "get_or_create_tuning_test_set",
     "get_tuning_case",
     "get_tuning_run",

apps/backend/src/rhesis/backend/app/services/metric_tuning/agreement.py

@@ -0,0 +1,74 @@
+"""Agreement: the share of judged cases the reviewer accepted.
+
+One number for a whole tuning test set, and the only one an author watches while
+editing an evaluation prompt -- change the wording, run again, see whether it
+went up.
+
+It was originally the share of cases where the metric's verdict equalled a stored
+expected verdict. There is no stored verdict any more (domain.local/adr/0005), so
+it means the same thing to its reader and is computed completely differently:
+accepted over accepted plus rejected, both of them a human's judgement.
+
+**The denominator is the whole design.** Every shortcut here inflates it. An
+unreviewed case counted as accepted makes a set nobody looked at report itself
+perfect; an errored case counted as rejected makes a flaky provider read as a bad
+metric. Both are left out of the ratio and reported beside it instead.
+
+Nothing is stored. The outcomes this folds over are themselves derived from the
+metric's current threshold on every read (``outcome.py``), so a review a run has
+just invalidated stops counting immediately rather than at the next write.
+"""
+
+import logging
+from collections import Counter
+from typing import Iterable
+
+from sqlalchemy.orm import Session
+
+from rhesis.backend.app import models
+from rhesis.backend.app.crud import metric_tuning as crud_metric_tuning
+from rhesis.backend.app.schemas.metric_tuning import TuningAgreement, TuningCaseOutcome
+from rhesis.backend.app.schemas.metric_tuning_metadata import parse_metric_tuning_case_metadata
+from rhesis.backend.app.services.metric_tuning.outcome import case_outcome
+from rhesis.backend.app.services.metric_tuning.test_sets import get_tuning_test_set
+
+logger = logging.getLogger(__name__)
+
+# Places to round the ratio to. Enough for any display, and it keeps two out of
+# three off the wire as 0.6667 rather than 0.6666666666666666.
+RATIO_PRECISION = 4
+
+
+def agreement_over(outcomes: Iterable[TuningCaseOutcome]) -> TuningAgreement:
+    """Fold case outcomes into the ratio and the counts that qualify it."""
+    counts = Counter(outcomes)
+    accepted = counts[TuningCaseOutcome.ACCEPTED]
+    rejected = counts[TuningCaseOutcome.REJECTED]
+    judged = accepted + rejected
+
+    return TuningAgreement(
+        # Nothing judged is no agreement, not perfect agreement.
+        ratio=round(accepted / judged, RATIO_PRECISION) if judged else None,
+        judged=judged,
+        accepted=accepted,
+        rejected=rejected,
+        unreviewed=counts[TuningCaseOutcome.UNREVIEWED],
+        errored=counts[TuningCaseOutcome.ERRORED],
+    )
+
+
+def get_agreement(db: Session, metric: models.Metric, organization_id: str) -> TuningAgreement:
+    """The metric's agreement as its stored reviews stand right now.
+
+    A metric with no tuning set has nothing to agree about, which is the same
+    all-zero, no-ratio answer as a set nobody has reviewed.
+    """
+    test_set = get_tuning_test_set(db, metric.id, organization_id)
+    if not test_set:
+        return TuningAgreement()
+
+    cases = crud_metric_tuning.get_tuning_cases(db, test_set.id, organization_id)
+    return agreement_over(
+        case_outcome(metric, parse_metric_tuning_case_metadata(db_test.test_metadata))[0]
+        for db_test in cases
+    )

apps/frontend/src/app/(protected)/metrics/[identifier]/tuning/MetricTuningTab.tsx

@@ -38,6 +38,7 @@ import { ApiClientFactory } from '@/utils/api-client/client-factory';
 import type { UUID } from 'crypto';
 import type { ScoreType } from '@/utils/api-client/interfaces/metric';
 import type {
+  MetricTuningAgreement,
   MetricTuningCase,
   MetricTuningCaseCreate,
   MetricTuningRun,
@@ -58,6 +59,9 @@ const INVALIDATED_HINT =
 const ERRORED_HINT =
   'The metric call failed for this case, so there is no verdict to judge.';
 
+const NO_AGREEMENT_HINT =
+  'Agreement is the share of judged cases you accepted. Nothing has been judged yet, so there is no share to report — a set nobody has looked at is not a set the metric got right.';
+
 /** Renders long free text on one line with the full value in a tooltip. */
 function TruncatedCell({ params }: { params: GridRenderCellParams }) {
   const value = typeof params.value === 'string' ? params.value : '';
@@ -248,6 +252,48 @@ function InvalidatedMark() {
   );
 }
 
+/**
+ * The metric's agreement, and the counts that stop it being read as more than
+ * it is.
+ *
+ * The denominator is the whole point. Unreviewed and errored cases are counted
+ * out of the ratio and reported beside it: counting either one in produces a
+ * plausible figure meaning something other than what its reader thinks — a set
+ * nobody looked at reading as perfect, or a flaky provider reading as a bad
+ * metric. The judged count sits next to the number for the same reason, so
+ * three out of three does not read like a solved problem.
+ */
+function AgreementSummary({ agreement }: { agreement: MetricTuningAgreement }) {
+  const { ratio, judged, unreviewed, errored } = agreement;
+  const total = judged + unreviewed + errored;
+
+  const qualifiers = [
+    judged > 0
+      ? `over ${judged} of ${total} ${total === 1 ? 'case' : 'cases'} judged`
+      : 'nothing judged yet',
+    unreviewed > 0 ? `${unreviewed} unreviewed` : null,
+    errored > 0 ? `${errored} the metric could not be reached on` : null,
+  ].filter(Boolean);
+
+  return (
+    <Box sx={{ display: 'flex', alignItems: 'baseline', gap: 1, mb: 0.5 }}>
+      <Typography variant="body2" color="text.secondary">
+        Agreement
+      </Typography>
+      <Typography
+        variant="h6"
+        component="span"
+        title={ratio === null ? NO_AGREEMENT_HINT : undefined}
+      >
+        {ratio === null ? '—' : `${Math.round(ratio * 100)}%`}
+      </Typography>
+      <Typography variant="body2" color="text.secondary">
+        {qualifiers.join(' · ')}
+      </Typography>
+    </Box>
+  );
+}
+
 /** One line about the latest run, or nothing when there has not been one. */
 function RunSummary({ run }: { run: MetricTuningRun | null }) {
   if (!run || run.status === 'never_run') return null;
@@ -443,6 +489,18 @@ export default function MetricTuningTab({ metricId }: MetricTuningTabProps) {
     [metricId, notifications]
   );
 
+  // Agreement is derived from the reviews, so judging a case moves it without a
+  // run. Re-read rather than recomputed here: one fold, on the server, is what
+  // stops the number and the rows it is folded from ever disagreeing.
+  const refreshRun = useCallback(async () => {
+    try {
+      const client = new ApiClientFactory().getMetricTuningClient();
+      setRun(await client.getTuningRun(metricId));
+    } catch {
+      // The line keeps its last value; the next review or poll corrects it.
+    }
+  }, [metricId]);
+
   /** Swaps in the case the review endpoint returns, leaving the rest alone. */
   const replaceCase = useCallback((updated: MetricTuningCase) => {
     setCases(prev =>
@@ -459,6 +517,7 @@ export default function MetricTuningTab({ metricId }: MetricTuningTabProps) {
             decision: 'accepted',
           })
         );
+        await refreshRun();
       } catch (error) {
         notifications.show(
           error instanceof Error
@@ -468,7 +527,7 @@ export default function MetricTuningTab({ metricId }: MetricTuningTabProps) {
         );
       }
     },
-    [metricId, notifications, replaceCase]
+    [metricId, notifications, refreshRun, replaceCase]
   );
 
   // Throws on failure so the dialog keeps the comment on screen — a rejection
@@ -483,15 +542,17 @@ export default function MetricTuningTab({ metricId }: MetricTuningTabProps) {
           comment,
         })
       );
+      await refreshRun();
     },
-    [metricId, rejecting, replaceCase]
+    [metricId, refreshRun, rejecting, replaceCase]
   );
 
   const handleAcceptRest = useCallback(async () => {
     setAcceptingRest(true);
     try {
       const client = new ApiClientFactory().getMetricTuningClient();
       setCases(await client.acceptRemainingTuningCases(metricId));
+      await refreshRun();
       notifications.show('Accepted every case left unreviewed', {
         severity: 'success',
         autoHideDuration: 4000,
@@ -506,7 +567,7 @@ export default function MetricTuningTab({ metricId }: MetricTuningTabProps) {
     } finally {
       setAcceptingRest(false);
     }
-  }, [metricId, notifications]);
+  }, [metricId, notifications, refreshRun]);
 
   const openAdd = useCallback(() => {
     setEditing(null);
@@ -707,6 +768,13 @@ export default function MetricTuningTab({ metricId }: MetricTuningTabProps) {
           />
         ) : (
           <>
+            {/* Nothing while a run is going: until the worker has cleared the
+                last run's results, the number is the previous run's, and one
+                sitting above a progress line reads as this run's. The progress
+                line says what is happening instead. */}
+            {hasResults && run && !isRunning && (
+              <AgreementSummary agreement={run.agreement} />
+            )}
             <RunSummary run={run} />
             <BaseDataGrid
               rows={cases as unknown as GridRowModel[]}