From 81f2008ef578fca98f7f0d7be91a51872db0d7a1 Mon Sep 17 00:00:00 2001 From: Rishab Ramanathan Date: Tue, 1 Sep 2026 19:08:24 -0700 Subject: [PATCH] fix(skill): document monitoring setup pitfalls --- .claude-plugin/marketplace.json | 2 +- .claude-plugin/plugin.json | 2 +- .cursor-plugin/plugin.json | 2 +- skills/openlayer/references/data-streaming.md | 7 +++++++ skills/openlayer/references/tests.md | 16 ++++++++++++++++ 5 files changed, 26 insertions(+), 3 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 99f6354..b7d697e 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -1,6 +1,6 @@ { "name": "openlayer", - "version": "0.4.0", + "version": "0.4.1", "description": "Openlayer agent skills marketplace \u2014 teach AI coding agents to integrate apps with Openlayer.", "owner": { "name": "Openlayer", diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index acd81a4..18821ab 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "openlayer", "description": "Skills for working with Openlayer, the AI evaluation and observability platform \u2014 tracing/monitoring, offline testing, evals, and CI/CD gates.", - "version": "0.4.0", + "version": "0.4.1", "author": { "name": "Openlayer", "email": "support@openlayer.com" diff --git a/.cursor-plugin/plugin.json b/.cursor-plugin/plugin.json index 5239e1b..624c287 100644 --- a/.cursor-plugin/plugin.json +++ b/.cursor-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "openlayer", "displayName": "Openlayer", - "version": "0.4.0", + "version": "0.4.1", "description": "Skills for working with Openlayer \u2014 the AI evaluation and observability platform for tracing/monitoring, offline testing, evals, and CI/CD gates.", "author": { "name": "Openlayer", diff --git a/skills/openlayer/references/data-streaming.md b/skills/openlayer/references/data-streaming.md index 5876af6..fb8a94d 100644 --- a/skills/openlayer/references/data-streaming.md +++ b/skills/openlayer/references/data-streaming.md @@ -31,6 +31,12 @@ For **traditional / tabular ML** (the `ConfigTabular*` classes, their required f Include an `inference_id` per row if you'll later add ground truth or update the row (`client.inference_pipelines.rows.update(...)`). +The configured source names describe the incoming row, but Openlayer persists mapped semantic fields +under canonical names. For example, an incoming `response` mapped as the output is stored as +`openlayer_output`. After streaming the first small batch, fetch one stored row and inspect its actual +schema before creating tests or filters. This catches mapping mistakes before they turn into skipped +evaluations. + ## Stream vs trace — don't double-publish If the app is already instrumented with `@trace` / a wrapper (`references/monitoring-instrumentation.md`), @@ -45,6 +51,7 @@ streaming *instead of* tracing for that path, not in addition. | Wrong config class for the task type | Validation error | Use the config matching the project `task_type` | | Streaming data that tracing already publishes | Duplicate rows | Pick one path per request flow | | No `inference_id` | Can't correlate later ground-truth updates | Set `inference_id_column_name` and provide stable ids | +| Authoring tests from upload field names | Mapped fields are persisted under canonical names | Inspect a stored row; target `openlayer_output` and the other `openlayer_*` fields | | Wrong timestamp unit/format | Rows mis-ordered or rejected | Use the unit the docs specify | | Row values are numpy scalars (from a pandas row) | Server dtype validation 400 (e.g. "must be int32/int64") | Cast to native Python types (`int(...)`, `.item()`) before streaming | | Guessing config field names | Runtime error | Confirm fields from the current docs/SDK | diff --git a/skills/openlayer/references/tests.md b/skills/openlayer/references/tests.md index 87397e4..5b0178c 100644 --- a/skills/openlayer/references/tests.md +++ b/skills/openlayer/references/tests.md @@ -24,6 +24,11 @@ Docs: `projects.tests.create`; also set `usesValidationDataset` / `usesTrainingDataset` / `usesMlModel`. - **Monitoring** tests run on live traces; set `evaluationWindow` / `delayWindow` (hours). +For monitoring-test API payloads, set `usesProductionData: true` and explicitly provide the other +applicable `uses*` booleans (`usesMlModel`, `usesReferenceDataset`, `usesTestResults`, +`usesTrainingDataset`, `usesValidationDataset`) rather than relying on omitted values. Some create +paths reject missing booleans instead of defaulting them. + ### Reference columns by their canonical `openlayer_*` names When a threshold targets a column (`column_name` insight param, or a `subpopulationFilters` @@ -56,6 +61,13 @@ name makes the test silently **SKIPPED** ("column not in dataset"), which a "0 f it. The backend rejects an unsupported subtype×task at sync. - **A single bad test fails the WHOLE push at sync** (Total Tests = 0) — you only get per-test status once sync passes. So fix sync-time rejections first, *then* read per-test pass/fail/skip. +- **Monitoring evaluation is asynchronous and time-windowed.** Rows must have timestamps inside the + evaluation window. After creating a set of monitoring tests, evaluate the inference pipeline, poll + the returned background task, and verify each test has a result. Creating the tests alone does not + make result or failed-row workflows ready. +- **Fixing a skipped test does not erase its old result.** Correct the definition and reevaluate with + overwrite when supported. If an obsolete definition or stale skipped result still wins default list + ordering, archive that test and create a clean replacement so users do not keep selecting it. ## Catalog at a glance @@ -114,7 +126,11 @@ MCP is connected (covered in a separate skill). | ------- | ------- | --- | | Inventing a `subtype`/threshold shape | Won't sync or evaluate | Copy from the catalog page (`tests/catalog/.md`) | | Targeting raw `output` instead of `openlayer_output` | Test silently SKIPPED | Use canonical `openlayer_*` column names | +| Omitting monitoring `uses*` flags | Test creation can fail validation | Set `usesProductionData: true` and provide the remaining applicable booleans explicitly | | Trusting "0 failing" totals | A skipped test isn't failing | Check per-test status after a push | +| Creating monitoring tests without running evaluation | Result and failed-row queries return nothing | Evaluate the pipeline, poll completion, and verify every active test result | +| Streaming rows outside the evaluation window | Test is empty or skipped | Use valid recent timestamps and confirm data-source health before evaluation | +| Leaving a corrected test's obsolete copy active | Default lists can surface stale skipped results | Archive the obsolete test after the corrected result is verified | | Wrong `type` for the metric | Misclassified test | Match `performance`/`integrity`/`consistency` to the catalog entry | | SQL test not selecting from `df` / returning many values | Test errors | Query `FROM df`, return one number | | Rubric test missing `criteria_list` params | Judge can't run | Provide the criteria per the catalog page |