From 86832784d3e2d3223d24b186361cf241b8016869 Mon Sep 17 00:00:00 2001 From: Grady Dillon Date: Mon, 20 Jul 2026 16:43:23 -0400 Subject: [PATCH 1/8] Add compliance overrides, scoped-key CRUD, register, and search Mirrors the Python SDK so both stay in lockstep with the API: - Compliance policy overrides (allowDomains / denyPatterns / respectRobots) on extract/bulk/crawl. JSON.stringify drops undefined, so unset overrides are omitted and can never clobber the key's policy. - Scoped API keys: createKey / listKeys / revokeKey, plus getLogs, with CreatedKey / ApiKeyInfo / RevokedKey / LogEntry / LogsPage. - Static register(): self-provision a free, extract-only key. - search(): web search plus extraction in one synchronous call, with SearchResult / SearchResults carrying a per-page status. Error mapping is by HTTP status, so the new API error codes needed no new error classes. Co-Authored-By: Claude Opus 4.8 --- README.md | 4 +- src/client.ts | 503 ++++++++++++++++++++++++++++++++++++++----- src/errors.ts | 1 - src/index.ts | 13 ++ src/models.ts | 221 ++++++++++++++++++- tests/client.test.ts | 435 +++++++++++++++++++++++++++++++++++++ 6 files changed, 1117 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index a363153..b5b294e 100644 --- a/README.md +++ b/README.md @@ -273,7 +273,7 @@ try { if (err instanceof RateLimitError) { console.log(`Quota hit. Resets in ${err.retryAfter}s.`); } else if (err instanceof UnprocessableEntityError) { - // err.code is one of: no_content, target_timeout, js_rendering_disabled, ... + // err.code is one of: no_content, target_timeout, ... console.log(`Extraction failed (${err.code}): ${err.message}`); } else { throw err; @@ -286,7 +286,7 @@ try { | `AuthenticationError` | 401 | `missing_api_key`, `invalid_api_key` | | `PermissionDeniedError` | 403 | `account_inactive`, `plan_not_supported`, `forbidden` | | `NotFoundError` | 404 | `job_not_found` | -| `UnprocessableEntityError` | 422 | `no_content`, `target_timeout`, `js_rendering_disabled`, `bulk_cap_exceeded`, `crawl_depth_exceeded` | +| `UnprocessableEntityError` | 422 | `no_content`, `target_timeout`, `bulk_cap_exceeded`, `crawl_depth_exceeded` | | `RateLimitError` | 429 | `rate_limit_too_fast` *(per-second cap; `retryAfterMs` carries the sub-second back-off)* · `rate_limit_exceeded` *(monthly quota; `retryAfter` in seconds)* | | `InternalServerError` | 5xx | — | | `APIConnectionError` | — | DNS / TCP / TLS / timeout failures, raised before any HTTP round-trip | diff --git a/src/client.ts b/src/client.ts index 70077bc..c634b1a 100644 --- a/src/client.ts +++ b/src/client.ts @@ -21,23 +21,38 @@ import { fromResponse, } from "./errors.js"; import { + type ApiKeyInfo, type BulkJob, type CrawlJob, + type CreatedKey, type ExtractResult, + type LogsPage, + type RegisteredAccount, + type RevokedKey, type RotatedKey, type RotatedWebhookSecret, + type SearchResults, type Usage, + apiKeyInfoFromDict, bulkJobFromResponse, crawlJobFromResponse, + createdKeyFromResponse, extractResultFromResponse, + logsPageFromResponse, + registeredAccountFromResponse, + revokedKeyFromResponse, rotatedKeyFromResponse, rotatedWebhookSecretFromResponse, + searchResultsFromResponse, usageFromResponse, } from "./models.js"; import { VERSION } from "./version.js"; const DEFAULT_BASE_URL = "https://api.wellmarked.io"; const DEFAULT_TIMEOUT_MS = 30_000; +// 3 attempts total. Enough to ride out a blip; low enough that a genuinely +// down API surfaces quickly instead of stalling the caller for a minute. +const DEFAULT_MAX_RETRIES = 2; const RESERVED_HEADERS = new Set([ "authorization", @@ -53,8 +68,29 @@ export interface WellMarkedOptions { apiKey?: string; /** API base URL. Override for testing. */ baseUrl?: string; - /** Per-request timeout, milliseconds. Defaults to 30000 (30s). */ + /** + * Timeout for a single attempt, milliseconds. Defaults to 30000 (30s). + * + * This is **per attempt, not per call**. With the default `maxRetries: 2`, + * a retryable request can therefore take up to roughly + * `timeoutMs * 3` plus backoff (~92s at the defaults) before it gives up. + * Lower `maxRetries` if you need a tighter ceiling. + */ timeoutMs?: number; + /** + * How many times to retry a request the SDK knows is safe to replay. + * Defaults to 2 (so 3 attempts total). Set 0 to disable. + * + * Only connection failures and 5xx responses are retried, and only for + * requests that can be replayed safely: GETs, and POSTs carrying an + * `Idempotency-Key` (which `bulk()` and `crawl()` send automatically). + * A `POST /extract` is never retried — the API bills it on arrival, and a + * connection error can't tell you whether it arrived. + * + * Retries reuse the same Idempotency-Key, so the API replays the original + * job rather than enqueuing a second one. + */ + maxRetries?: number; /** * Bring your own `fetch`. Defaults to the global `fetch`. Useful for * polyfills, custom agents/proxies, or test mocking. @@ -70,14 +106,73 @@ export interface WellMarkedOptions { headers?: Record; } -export interface ExtractOptions { +/** + * Per-request compliance overrides, shared by extract / bulk / crawl. Each can + * only NARROW the API key's own policy server-side — add denies, restrict to a + * subset of the key's allowed domains, or upgrade robots to strict — never + * widen it. An omitted field leaves the key's policy untouched. + */ +export interface PolicyOverrideOptions { + /** Restrict this request to these domains (and their subdomains). */ + allowDomains?: string[]; + /** Extra deny globs for this request (matched on hostname and full URL). */ + denyPatterns?: string[]; + /** `"strict"` or `"lax"`; can tighten but not loosen the key's setting. */ + respectRobots?: "strict" | "lax"; +} + +export interface ExtractOptions extends PolicyOverrideOptions { /** - * Use Playwright to render JS-heavy pages. Requires a Pro/Enterprise - * plan AND `ENABLE_JS_RENDERING=true` on the API instance. + * Use Playwright to render JS-heavy pages. Requires a Pro, Growth, or + * Enterprise plan; Free returns `plan_not_supported`. */ renderJs?: boolean; } +/** Options for `search`. */ +export interface SearchOptions { + /** + * How many results to fetch + extract. Clamped to 1..10 server-side (a + * search is one synchronous call). Default 5. + */ + numResults?: number; + /** Render JS-heavy result pages before extracting. */ + renderJs?: boolean; +} + +/** Options for `createKey`. */ +export interface CreateKeyOptions { + /** A label for the key (shown in `listKeys`). Defaults to "default". */ + name?: string; +} + +/** Options for the static `WellMarked.register`. */ +export interface RegisterOptions { + /** API base URL. Override for testing. */ + baseUrl?: string; + /** Timeout for the single request, milliseconds. Defaults to 30000. */ + timeoutMs?: number; + /** Bring your own `fetch`. Defaults to the global `fetch`. */ + fetch?: typeof fetch; +} + +/** Options for `getLogs`. */ +export interface GetLogsOptions { + /** Page size, 1–200. Defaults to 50. */ + limit?: number; + /** Row offset for pagination. Defaults to 0. */ + offset?: number; +} + +/** Build the snake_case policy-override body fields, omitting unset ones. */ +function policyOverrides(o: PolicyOverrideOptions): Record { + const out: Record = {}; + if (o.allowDomains !== undefined) out.allow_domains = o.allowDomains; + if (o.denyPatterns !== undefined) out.deny_patterns = o.denyPatterns; + if (o.respectRobots !== undefined) out.respect_robots = o.respectRobots; + return out; +} + /** * Options shared by `bulk` and `crawl` for opting in to a job.completed * webhook delivery in lieu of polling. @@ -100,13 +195,36 @@ export interface JobWebhookOptions { * `results_url` pointing back at `getJob`. */ webhookIncludeResults?: boolean; + /** + * Sent as the `Idempotency-Key` header, so a replayed submission returns + * the ORIGINAL job instead of enqueuing a second one and charging your + * quota twice. + * + * A key is generated automatically and reused across the SDK's own internal + * retries (see `maxRetries`), which covers the common case of a connection + * blip mid-submission. + * + * **Pass your own key to survive anything the SDK can't retry for you** — + * a process crash, or your code catching the error and calling `bulk()` + * again. That second call mints a *new* key and gets a *second* job unless + * you reuse a stable one. + * + * Same key + same body replays the original job. Same key + a *different* + * body throws `UnprocessableEntityError` (`idempotency_key_reuse`) — use a + * fresh key per distinct submission. Records expire after 6 hours, matching + * the job TTL. + * + * Note the body must match byte-for-byte: reordering `urls` counts as a + * different request. + */ + idempotencyKey?: string; } -export interface BulkOptions extends JobWebhookOptions { +export interface BulkOptions extends JobWebhookOptions, PolicyOverrideOptions { renderJs?: boolean; } -export interface CrawlOptions extends JobWebhookOptions { +export interface CrawlOptions extends JobWebhookOptions, PolicyOverrideOptions { /** Max BFS depth from the root. Defaults to 1. Must be >= 0. */ depth?: number; renderJs?: boolean; @@ -145,16 +263,37 @@ function defaultHeaders(apiKey: string): Record { function mergeHeaders( apiKey: string, extra: Record | undefined, + perRequest?: Record, ): Record { const out = defaultHeaders(apiKey); - if (!extra) return out; - for (const [k, v] of Object.entries(extra)) { - if (RESERVED_HEADERS.has(k.toLowerCase())) continue; - out[k] = v; + // Client-wide headers first, then per-request ones — a header passed to a + // single call wins over the same header set on the client. Both are filtered + // against RESERVED_HEADERS: a stray Authorization from either source would + // break rotateKey() mid-session (see RESERVED_HEADERS). + for (const source of [extra, perRequest]) { + if (!source) continue; + for (const [k, v] of Object.entries(source)) { + if (RESERVED_HEADERS.has(k.toLowerCase())) continue; + out[k] = v; + } } return out; } +/** + * Idempotency keys must be unique per logical operation, which is exactly why + * they can't ride on the client-wide `headers` option: that would pin every + * subsequent call to the same key. Hence the per-request channel above. + */ +function newIdempotencyKey(): string { + const g = globalThis as { crypto?: { randomUUID?: () => string } }; + if (typeof g.crypto?.randomUUID === "function") return g.crypto.randomUUID(); + // Node 18.17+ and every modern browser have crypto.randomUUID. This is a + // last resort for exotic runtimes; uniqueness only has to hold within one + // caller's retry window, not globally. + return `wm-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 12)}`; +} + function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } @@ -170,6 +309,7 @@ export class WellMarked { private apiKey: string; private readonly baseUrl: string; private readonly timeoutMs: number; + private readonly maxRetries: number; private readonly fetchImpl: typeof fetch; private readonly extraHeaders: Record; @@ -177,6 +317,7 @@ export class WellMarked { this.apiKey = resolveApiKey(options.apiKey); this.baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, ""); this.timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; + this.maxRetries = Math.max(0, options.maxRetries ?? DEFAULT_MAX_RETRIES); const f = options.fetch ?? (typeof fetch !== "undefined" ? fetch : undefined); if (!f) { throw new Error( @@ -195,6 +336,90 @@ export class WellMarked { } } + // ── Self-registration ─────────────────────────────────────────────────────── + + /** + * Self-register for a free, extract-only API key — no existing key needed. + * + * The zero-to-first-call path for an agent that discovered WellMarked + * programmatically. Returns a `RegisteredAccount` whose `apiKey` is a weak + * credential (Free plan, `scopes: ["extract"]`); build a client with it: + * + * const account = await WellMarked.register("agent@example.com"); + * const wm = new WellMarked({ apiKey: account.apiKey }); + * await wm.extract("https://example.com"); + * + * Not retried — registration mints an account and isn't idempotent. + * + * Throws: + * - `RateLimitError` — `register_rate_limited`: too many from this IP. + * - `APIStatusError` — `email_taken` (409), or `service_unavailable` (503) + * if the limiter backend is momentarily down. + */ + static async register( + email: string, + options: RegisterOptions = {}, + ): Promise { + const baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, ""); + const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; + const f = options.fetch ?? (typeof fetch !== "undefined" ? fetch : undefined); + if (!f) { + throw new Error( + "No fetch implementation available. Pass `fetch:` to register() " + + "(undici, node-fetch, etc.) or upgrade to Node 18+.", + ); + } + const fetchImpl = f.bind(globalThis) as typeof fetch; + + let controller: AbortController | null = null; + let timer: ReturnType | null = null; + const init: RequestInitWithSignal = { + method: "POST", + headers: { "Content-Type": "application/json", Accept: "application/json" }, + body: JSON.stringify({ email }), + }; + if (timeoutMs > 0 && typeof AbortController !== "undefined") { + controller = new AbortController(); + init.signal = controller.signal; + timer = setTimeout(() => controller!.abort(), timeoutMs); + } + + let response: Response; + try { + response = await fetchImpl(`${baseUrl}/register`, init as RequestInit); + } catch (err) { + throw new APIConnectionError( + `Could not reach the WellMarked API: ${stringifyError(err)}`, + { cause: err }, + ); + } finally { + if (timer !== null) clearTimeout(timer); + } + + let bodyText = ""; + try { + bodyText = await response.text(); + } catch (err) { + throw new APIConnectionError( + `Could not read API response body: ${stringifyError(err)}`, + { cause: err }, + ); + } + let body: unknown = null; + if (bodyText.length > 0) { + try { + body = JSON.parse(bodyText); + } catch { + body = null; + } + } + const data = parseResponse(response.status, body, response.headers) as Record< + string, + unknown + >; + return registeredAccountFromResponse(data); + } + // ── Endpoints ────────────────────────────────────────────────────────────── /** @@ -202,27 +427,63 @@ export class WellMarked { * * Throws: * - `RateLimitError` — monthly plan limit reached. - * - `UnprocessableEntityError` — `no_content`, `target_timeout`, or - * `js_rendering_disabled`. + * - `PermissionDeniedError` — `plan_not_supported`: `renderJs=true` + * on the Free plan. + * - `UnprocessableEntityError` — `no_content` or `target_timeout`. * - `AuthenticationError` — missing or invalid API key. */ async extract(url: string, options: ExtractOptions = {}): Promise { const body = await this.request("POST", "/extract", { url, render_js: options.renderJs === true, + ...policyOverrides(options), }); return extractResultFromResponse(body as Record); } + /** + * Search the web and extract each result to Markdown. Synchronous — one + * round trip, no job to poll. + * + * The query runs against the search provider and the result pages are + * extracted concurrently, returned together with a per-page `status` (a slow + * or blocked page becomes an error item, never sinks the call). Costs + * `1 + results.length` requests — one for the query, one per page returned. + * + * Throws: + * - `PermissionDeniedError` — `plan_not_supported`: search requires a + * Pro, Growth, or Enterprise plan. + * - `RateLimitError` — would exceed remaining monthly quota. + * - `InternalServerError` — the provider is unconfigured or + * unreachable (`code === "service_unavailable"`). + */ + async search(query: string, options: SearchOptions = {}): Promise { + const body = await this.request("POST", "/search", { + query, + num_results: options.numResults ?? 5, + render_js: options.renderJs === true, + }); + return searchResultsFromResponse(body as Record); + } + /** * Submit a batch of URLs for concurrent extraction. * * Returns immediately with `status="queued"`. Poll with `getJob` or * block with `waitForJob` to collect results. * + * An `Idempotency-Key` is generated per call and reused across the SDK's + * internal retries, so a connection blip replays the original job rather + * than enqueuing a second one. To be safe against retries *your own code* + * makes, pass a stable `idempotencyKey` — see `BulkOptions.idempotencyKey`. + * * Throws: - * - `PermissionDeniedError` — `plan_not_supported` (Free tier). - * - `UnprocessableEntityError` — `bulk_cap_exceeded` (50 on Pro, 200 on Growth). + * - `PermissionDeniedError` — `plan_not_supported`: `renderJs=true` + * on the Free plan. + * - `UnprocessableEntityError` — `bulk_cap_exceeded` (5 on Free, 50 on + * Pro, 200 on Growth), or + * `idempotency_key_reuse` if the key was + * already used for a different body. * - `RateLimitError` — would exceed remaining monthly quota. */ async bulk(urls: Iterable, options: BulkOptions = {}): Promise { @@ -233,12 +494,15 @@ export class WellMarked { const payload: Record = { urls: urlList, render_js: options.renderJs === true, + ...policyOverrides(options), }; if (options.webhookUrl !== undefined) { payload.webhook_url = options.webhookUrl; payload.webhook_include_results = options.webhookIncludeResults === true; } - const body = await this.request("POST", "/bulk", payload); + const body = await this.request("POST", "/bulk", payload, { + "Idempotency-Key": options.idempotencyKey ?? newIdempotencyKey(), + }); return bulkJobFromResponse(body as Record); } @@ -340,19 +604,27 @@ export class WellMarked { url, depth, render_js: options.renderJs === true, + ...policyOverrides(options), }; if (options.webhookUrl !== undefined) { payload.webhook_url = options.webhookUrl; payload.webhook_include_results = options.webhookIncludeResults === true; } - const body = await this.request("POST", "/crawl", payload); + const body = await this.request("POST", "/crawl", payload, { + "Idempotency-Key": options.idempotencyKey ?? newIdempotencyKey(), + }); return crawlJobFromResponse(body as Record); } // ── Custom headers ───────────────────────────────────────────────────────── /** - * Add or replace a per-request header for the rest of this client's life. + * Add or replace a header for the rest of this client's life. + * + * Client-wide, so it is the wrong place for `Idempotency-Key`: that must be + * unique per submission, and pinning one here would make every later + * `bulk()` / `crawl()` replay the first job. Pass `idempotencyKey` to those + * methods instead (they generate one per call by default). * * Authorization / Content-Type / Accept are reserved — calls that try * to set those are silently ignored. To rotate the bearer token, use @@ -414,6 +686,72 @@ export class WellMarked { return rotatedWebhookSecretFromResponse(body as Record); } + // ── Key management (scoped keys) ───────────────────────────────────────────── + + /** + * Mint a new scoped API key on this account. + * + * `scopes` is a non-empty subset of `extract`, `bulk`, `crawl`, `keys`; the + * calling key can only grant scopes it holds. The raw key is in the returned + * `apiKey` — store it, it's shown once. Requires the `keys` scope. Does not + * count toward your quota. + * + * Throws: + * - `PermissionDeniedError` — `insufficient_scope`: your key lacks + * `keys`, or a scope it doesn't hold. + * - `UnprocessableEntityError` — `invalid_request`: empty/unknown scopes. + */ + async createKey(scopes: string[], options: CreateKeyOptions = {}): Promise { + const body = await this.request("POST", "/keys", { + scopes, + name: options.name ?? "default", + }); + return createdKeyFromResponse(body as Record); + } + + /** + * List this account's keys (metadata only — never the raw values), including + * revoked ones (`revokedAt` set). Requires the `keys` scope. Does not count + * toward your quota. + */ + async listKeys(): Promise { + const body = (await this.request("GET", "/keys")) as Record; + const raw = Array.isArray(body.keys) ? body.keys : []; + return raw + .filter((k): k is Record => k !== null && typeof k === "object") + .map(apiKeyInfoFromDict); + } + + /** + * Revoke a key by id — it stops authenticating immediately. Idempotent. + * Requires the `keys` scope. Does not count toward your quota. + * + * Throws `NotFoundError` (`key_not_found`) if no such key exists on this + * account. + */ + async revokeKey(keyId: string): Promise { + const body = await this.request("DELETE", `/keys/${keyId}`); + return revokedKeyFromResponse(body as Record); + } + + // ── Audit log ──────────────────────────────────────────────────────────────── + + /** + * Return this account's request history, newest first — the audit trail + * (which key ran each call and how its policy decided). Own rows only. + * Paginate via `offset += limit` while `hasMore` is true. Does not count + * toward your quota. + */ + async getLogs(options: GetLogsOptions = {}): Promise { + const limit = options.limit ?? 50; + const offset = options.offset ?? 0; + const body = await this.request( + "GET", + `/logs?limit=${encodeURIComponent(limit)}&offset=${encodeURIComponent(offset)}`, + ); + return logsPageFromResponse(body as Record); + } + /** * Internal: read the current API key. Exposed for tests. * Not part of the public, semver-stable surface. @@ -428,58 +766,113 @@ export class WellMarked { method: string, path: string, json?: unknown, + perRequestHeaders?: Record, ): Promise { const url = `${this.baseUrl}${path}`; - const headers = mergeHeaders(this.apiKey, this.extraHeaders); + const headers = mergeHeaders(this.apiKey, this.extraHeaders, perRequestHeaders); + + // Only replay a request when replaying it is actually safe. + // + // A connection error is ambiguous: the request may well have reached the + // API and been executed — we just never saw the response. Blindly retrying + // a POST /extract would extract (and bill) twice. GETs are naturally safe, + // and a POST is safe exactly when it carries an Idempotency-Key, because + // then the API replays the original job instead of creating a second one. + // + // This is what makes the auto-generated key on bulk()/crawl() worth + // anything: the key is reused across THESE attempts, so all of them + // collapse to one job. A caller who catches an error and calls bulk() + // again still mints a new key and gets a second job — that's why the + // docs tell you to pass your own key for retries across calls. + // + // Note this inspects `perRequestHeaders`, NOT the merged set. + // Idempotency-Key is not a RESERVED_HEADER, so a caller can legally + // `setHeader("Idempotency-Key", ...)` client-wide — and reading the merged + // headers would then mark POST /extract replay-safe and retry it, billing + // twice. Only a key this call actually attached counts. + const safeToReplay = + method.toUpperCase() === "GET" || + Object.keys(perRequestHeaders ?? {}).some( + (h) => h.toLowerCase() === "idempotency-key", + ); + const attempts = safeToReplay ? this.maxRetries + 1 : 1; - const init: RequestInitWithSignal = { method, headers }; - if (json !== undefined) { - init.body = JSON.stringify(json); - } + let lastError: unknown; + for (let attempt = 0; attempt < attempts; attempt++) { + if (attempt > 0) await sleep(backoffMs(attempt)); - let controller: AbortController | null = null; - let timer: ReturnType | null = null; - if (this.timeoutMs > 0 && typeof AbortController !== "undefined") { - controller = new AbortController(); - init.signal = controller.signal; - timer = setTimeout(() => controller!.abort(), this.timeoutMs); - } + const init: RequestInitWithSignal = { method, headers }; + if (json !== undefined) { + init.body = JSON.stringify(json); + } - let response: Response; - try { - response = await this.fetchImpl(url, init as RequestInit); - } catch (err) { - throw new APIConnectionError( - `Could not reach the WellMarked API: ${stringifyError(err)}`, - { cause: err }, - ); - } finally { - if (timer !== null) clearTimeout(timer); - } + // A fresh controller per attempt: reusing an aborted signal would make + // every retry fail instantly. + let controller: AbortController | null = null; + let timer: ReturnType | null = null; + if (this.timeoutMs > 0 && typeof AbortController !== "undefined") { + controller = new AbortController(); + init.signal = controller.signal; + timer = setTimeout(() => controller!.abort(), this.timeoutMs); + } - let bodyText = ""; - try { - bodyText = await response.text(); - } catch (err) { - throw new APIConnectionError( - `Could not read API response body: ${stringifyError(err)}`, - { cause: err }, - ); - } + let response: Response; + try { + response = await this.fetchImpl(url, init as RequestInit); + } catch (err) { + lastError = new APIConnectionError( + `Could not reach the WellMarked API: ${stringifyError(err)}`, + { cause: err }, + ); + continue; + } finally { + if (timer !== null) clearTimeout(timer); + } - let body: unknown = null; - if (bodyText.length > 0) { + // 5xx is the other ambiguous case — the API may have committed the job + // before failing. 4xx is deterministic: replaying reproduces it, so + // don't waste the caller's time. + if (response.status >= 500 && attempt < attempts - 1) { + lastError = undefined; + continue; + } + + let bodyText = ""; try { - body = JSON.parse(bodyText); - } catch { - body = null; + bodyText = await response.text(); + } catch (err) { + throw new APIConnectionError( + `Could not read API response body: ${stringifyError(err)}`, + { cause: err }, + ); } + + let body: unknown = null; + if (bodyText.length > 0) { + try { + body = JSON.parse(bodyText); + } catch { + body = null; + } + } + + return parseResponse(response.status, body, response.headers); } - return parseResponse(response.status, body, response.headers); + // Exhausted the retries on a connection error. A 5xx that ran out of + // attempts fell through to parseResponse above and threw there. + throw lastError ?? new APIConnectionError("Could not reach the WellMarked API."); } } +/** Exponential backoff with jitter. Jitter matters when an agent fans out — + * without it, N clients retry in lockstep and hit the API as a thundering + * herd at exactly the moment it is already struggling. */ +function backoffMs(attempt: number): number { + const base = Math.min(500 * 2 ** (attempt - 1), 4000); + return base + Math.random() * 250; +} + function parseResponse( statusCode: number, body: unknown, diff --git a/src/errors.ts b/src/errors.ts index fdd3c3d..78e6075 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -102,7 +102,6 @@ export class NotFoundError extends APIStatusError { * Common `code` values: * - `no_content` — could not identify main content on the page * - `target_timeout` — the target URL timed out - * - `js_rendering_disabled` — `renderJs=true` but the server has it off * - `bulk_cap_exceeded` — more URLs than the plan allows per request * - `crawl_depth_exceeded` — requested depth above the plan cap */ diff --git a/src/index.ts b/src/index.ts index 406ff49..546fa15 100644 --- a/src/index.ts +++ b/src/index.ts @@ -14,9 +14,14 @@ export { WellMarked, type WellMarkedOptions, type ExtractOptions, + type SearchOptions, type BulkOptions, type CrawlOptions, type JobWebhookOptions, + type PolicyOverrideOptions, + type CreateKeyOptions, + type GetLogsOptions, + type RegisterOptions, type WaitForJobOptions, } from "./client.js"; export { @@ -26,15 +31,23 @@ export { type WebhookPayload, } from "./webhooks.js"; export { + type ApiKeyInfo, type BulkItem, type BulkJob, type CrawlItem, type CrawlJob, + type CreatedKey, type ExtractionMeta, type ExtractResult, type JobStatus, + type LogEntry, + type LogsPage, + type RegisteredAccount, + type RevokedKey, type RotatedKey, type RotatedWebhookSecret, + type SearchResult, + type SearchResults, type TruncatedReason, type Usage, isBulkJob, diff --git a/src/models.ts b/src/models.ts index 2808553..3a76356 100644 --- a/src/models.ts +++ b/src/models.ts @@ -63,14 +63,81 @@ export function extractResultFromResponse(body: Record): Extrac }; } +// ── Search ─────────────────────────────────────────────────────────────────── + +/** + * One result in a `SearchResults`. + * + * On success `status === "ok"` and `markdown` is populated; on a per-page + * failure `status === "error"` and `error` carries a stable code (same + * convention as `BulkItem.error`). `title` is the extracted title on success, + * else the search provider's; `snippet` is always the provider's result + * snippet, so a page that failed extraction still carries context. + */ +export interface SearchResult { + url: string; + status: "ok" | "error"; + title: string | null; + snippet: string | null; + markdown: string | null; + error: string | null; + /** True when `status === "ok"`. */ + readonly ok: boolean; +} + +export function searchResultFromDict(data: Record): SearchResult { + const url = typeof data.url === "string" ? data.url : ""; + const status: "ok" | "error" = data.status === "ok" ? "ok" : "error"; + const title = typeof data.title === "string" ? data.title : null; + const snippet = typeof data.snippet === "string" ? data.snippet : null; + const markdown = typeof data.markdown === "string" ? data.markdown : null; + const error = typeof data.error === "string" ? data.error : null; + return { + url, + status, + title, + snippet, + markdown, + error, + get ok(): boolean { + return this.status === "ok"; + }, + }; +} + +/** + * Result of `POST /search` — the query plus the extracted result pages. + * Synchronous: unlike bulk/crawl there is no job to poll; `results` is already + * populated, with partial failures marked per item. + */ +export interface SearchResults { + query: string; + results: SearchResult[]; + requestId: string; +} + +export function searchResultsFromResponse(body: Record): SearchResults { + const rawResults = Array.isArray(body.results) ? body.results : []; + const results = rawResults + .filter((r): r is Record => r !== null && typeof r === "object") + .map(searchResultFromDict); + return { + query: typeof body.query === "string" ? body.query : "", + results, + requestId: typeof body.request_id === "string" ? body.request_id : "", + }; +} + // ── Bulk ───────────────────────────────────────────────────────────────────── /** * One entry in a bulk job's `results` list. * * On success, `markdown` and `metadata` are populated and `error` is null. - * On a per-URL failure, `markdown`/`metadata` are null and `error` carries - * an API error code (e.g. `target_timeout`). + * On a per-URL failure, `markdown`/`metadata` are null and `error` carries a + * stable API error *code* — never a human message — e.g. `target_timeout`, + * `domain_denied`, `internal_error`. Same convention as `CrawlItem.error`, so + * results parse identically on either endpoint. */ export interface BulkItem { url: string; @@ -356,6 +423,156 @@ export function rotatedWebhookSecretFromResponse( }; } +// ── Self-registration ──────────────────────────────────────────────────────── + +/** + * Result of `WellMarked.register` (`POST /register`). + * + * `apiKey` is the new raw key — shown once, store it. The account is + * deliberately weak: `plan === "free"` and `scopes === ["extract"]`. Build a + * client with it via `new WellMarked({ apiKey: account.apiKey })`. + */ +export interface RegisteredAccount { + apiKey: string; + userId: string; + plan: string; + scopes: string[]; +} + +export function registeredAccountFromResponse( + body: Record, +): RegisteredAccount { + return { + apiKey: typeof body.api_key === "string" ? body.api_key : "", + userId: typeof body.user_id === "string" ? body.user_id : "", + plan: typeof body.plan === "string" ? body.plan : "", + scopes: Array.isArray(body.scopes) ? (body.scopes as string[]) : [], + }; +} + +// ── Key management (scoped keys) ───────────────────────────────────────────── + +/** + * Result of `createKey` (`POST /keys`). `apiKey` is the new raw key — shown + * once, store it before discarding this object. `scopes` is the subset it was + * granted (extract / bulk / crawl / keys). + */ +export interface CreatedKey { + id: string; + apiKey: string; + name: string; + scopes: string[]; + createdAt: Date | null; +} + +export function createdKeyFromResponse(body: Record): CreatedKey { + return { + id: typeof body.id === "string" ? body.id : "", + apiKey: typeof body.api_key === "string" ? body.api_key : "", + name: typeof body.name === "string" ? body.name : "", + scopes: Array.isArray(body.scopes) ? (body.scopes as string[]) : [], + createdAt: parseDate(body.created_at), + }; +} + +/** + * One key's metadata from `listKeys` (`GET /keys`). Never carries the raw key. + * `revokedAt` is set once the key has been revoked; `active` is a convenience. + */ +export interface ApiKeyInfo { + id: string; + name: string; + scopes: string[]; + createdAt: Date | null; + revokedAt: Date | null; + readonly active: boolean; +} + +export function apiKeyInfoFromDict(data: Record): ApiKeyInfo { + return { + id: typeof data.id === "string" ? data.id : "", + name: typeof data.name === "string" ? data.name : "", + scopes: Array.isArray(data.scopes) ? (data.scopes as string[]) : [], + createdAt: parseDate(data.created_at), + revokedAt: parseDate(data.revoked_at), + get active(): boolean { + return this.revokedAt === null; + }, + }; +} + +/** Result of `revokeKey` (`DELETE /keys/{id}`). */ +export interface RevokedKey { + id: string; + revokedAt: Date | null; +} + +export function revokedKeyFromResponse(body: Record): RevokedKey { + return { + id: typeof body.id === "string" ? body.id : "", + revokedAt: parseDate(body.revoked_at), + }; +} + +// ── Audit log ──────────────────────────────────────────────────────────────── + +/** + * One row of your request history from `getLogs` (`GET /logs`). + * + * `policyDecision` records how the key's compliance policy decided: + * `"allowed"` | `"domain_not_allowed"` | `"domain_denied"` | + * `"robots_disallowed"`. `keyId` attributes the request to a key. + */ +export interface LogEntry { + id: string; + timestamp: Date | null; + targetUrl: string; + statusCode: number; + durationMs: number; + errorCode: string | null; + renderJs: boolean | null; + keyId: string | null; + policyDecision: string | null; +} + +export function logEntryFromDict(data: Record): LogEntry { + return { + id: typeof data.id === "string" ? data.id : "", + timestamp: parseDate(data.timestamp), + targetUrl: typeof data.target_url === "string" ? data.target_url : "", + statusCode: typeof data.status_code === "number" ? data.status_code : 0, + durationMs: typeof data.duration_ms === "number" ? data.duration_ms : 0, + errorCode: typeof data.error_code === "string" ? data.error_code : null, + renderJs: typeof data.render_js === "boolean" ? data.render_js : null, + keyId: typeof data.key_id === "string" ? data.key_id : null, + policyDecision: + typeof data.policy_decision === "string" ? data.policy_decision : null, + }; +} + +/** + * One page of `getLogs` results. `hasMore` is true when further rows exist + * beyond this page — advance by `offset += limit`. + */ +export interface LogsPage { + logs: LogEntry[]; + limit: number; + offset: number; + hasMore: boolean; +} + +export function logsPageFromResponse(body: Record): LogsPage { + const rawLogs = Array.isArray(body.logs) ? body.logs : []; + return { + logs: rawLogs + .filter((r): r is Record => r !== null && typeof r === "object") + .map(logEntryFromDict), + limit: typeof body.limit === "number" ? body.limit : 0, + offset: typeof body.offset === "number" ? body.offset : 0, + hasMore: body.has_more === true, + }; +} + // ── Type guards ────────────────────────────────────────────────────────────── export function isBulkJob(job: BulkJob | CrawlJob): job is BulkJob { diff --git a/tests/client.test.ts b/tests/client.test.ts index c80ec54..6db119a 100644 --- a/tests/client.test.ts +++ b/tests/client.test.ts @@ -749,3 +749,438 @@ describe("request body", () => { }); }); }); + +// ── Idempotency-Key ──────────────────────────────────────────────────────── +// The header is what makes a retried /bulk replay the original job instead of +// double-charging the caller's quota. If the SDK silently stops sending it, +// the API's protection is inert and nothing else would catch it. + +const QUEUED_JOB = { + job_id: "1c4f9a02-0000-0000-0000-000000000000", + status: "queued", + total: 1, + completed: 0, + results: [], +}; + +describe("idempotency", () => { + it("bulk() sends a generated Idempotency-Key when none is given", async () => { + mock.on("POST", "/bulk", () => jsonResponse(200, QUEUED_JOB)); + const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + + await wm.bulk(["https://a.example"]); + + expect(mock.calls[0]!.headers["idempotency-key"]).toBeTruthy(); + }); + + it("bulk() honours an explicit key", async () => { + mock.on("POST", "/bulk", () => jsonResponse(200, QUEUED_JOB)); + const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + + await wm.bulk(["https://a.example"], { idempotencyKey: "caller-chosen" }); + + expect(mock.calls[0]!.headers["idempotency-key"]).toBe("caller-chosen"); + }); + + it("gives each submission a distinct generated key", async () => { + // Two submissions are two operations — sharing a key would make the + // second replay the first one's job. + mock.on("POST", "/bulk", () => jsonResponse(200, QUEUED_JOB)); + const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + + await wm.bulk(["https://a.example"]); + await wm.bulk(["https://b.example"]); + + expect(mock.calls[0]!.headers["idempotency-key"]).not.toBe( + mock.calls[1]!.headers["idempotency-key"], + ); + }); + + it("crawl() sends an Idempotency-Key", async () => { + mock.on("POST", "/crawl", () => + jsonResponse(200, { ...QUEUED_JOB, kind: "crawl", total: 0 }), + ); + const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + + await wm.crawl("https://a.example"); + + expect(mock.calls[0]!.headers["idempotency-key"]).toBeTruthy(); + }); + + it("a client-wide header cannot pin Idempotency-Key across submissions", async () => { + // setHeader is client-wide, so it's the wrong channel for idempotency. + // The per-request key must win, or every later bulk() would replay the + // first job. + mock.on("POST", "/bulk", () => jsonResponse(200, QUEUED_JOB)); + const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + wm.setHeader("Idempotency-Key", "pinned-forever"); + + await wm.bulk(["https://a.example"]); + await wm.bulk(["https://b.example"]); + + expect(mock.calls[0]!.headers["idempotency-key"]).not.toBe("pinned-forever"); + expect(mock.calls[0]!.headers["idempotency-key"]).not.toBe( + mock.calls[1]!.headers["idempotency-key"], + ); + }); + + it("per-request headers still cannot override Authorization", async () => { + mock.on("POST", "/bulk", () => jsonResponse(200, QUEUED_JOB)); + const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + + await wm.bulk(["https://a.example"], { idempotencyKey: "k1" }); + + expect(mock.calls[0]!.headers["authorization"]).toBe(`Bearer ${API_KEY}`); + }); +}); + +// ── Internal retry ───────────────────────────────────────────────────────── +// Retries exist so the auto-generated Idempotency-Key is worth something: a +// connection blip is ambiguous (the job may already exist), and replaying with +// the SAME key collapses the attempts into one job instead of two. + +describe("retry", () => { + it("retries a connection failure on bulk and reuses the same key", async () => { + let attempts = 0; + const seenKeys: string[] = []; + const failingFetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + attempts++; + const h = init!.headers as Record; + seenKeys.push(h["Idempotency-Key"]!); + if (attempts === 1) throw new TypeError("network down"); + return jsonResponse(200, QUEUED_JOB); + }) as typeof fetch; + + const wm = new WellMarked({ apiKey: API_KEY, fetch: failingFetch }); + const job = await wm.bulk(["https://a.example"]); + + expect(attempts).toBe(2); + expect(job.status).toBe("queued"); + // The whole point: both attempts carry the SAME key, so the API replays + // rather than creating a second job. + expect(seenKeys[0]).toBe(seenKeys[1]); + }); + + it("does NOT retry POST /extract — the API bills it on arrival", async () => { + // A connection error can't tell us whether the extraction happened. + // /extract takes no Idempotency-Key, so replaying could bill twice. + let attempts = 0; + const failingFetch = (async () => { + attempts++; + throw new TypeError("network down"); + }) as typeof fetch; + + const wm = new WellMarked({ apiKey: API_KEY, fetch: failingFetch }); + await expect(wm.extract("https://a.example")).rejects.toBeInstanceOf(APIConnectionError); + expect(attempts).toBe(1); + }); + + it("retries 5xx but not 4xx", async () => { + let attempts = 0; + const flaky = (async () => { + attempts++; + if (attempts === 1) return jsonResponse(503, { error: { code: "x", message: "down" } }); + return jsonResponse(200, QUEUED_JOB); + }) as typeof fetch; + const wm = new WellMarked({ apiKey: API_KEY, fetch: flaky }); + await wm.bulk(["https://a.example"]); + expect(attempts).toBe(2); + + let attempts4xx = 0; + const deterministic = (async () => { + attempts4xx++; + return jsonResponse(422, { + error: { code: "bulk_cap_exceeded", message: "too many" }, + }); + }) as typeof fetch; + const wm2 = new WellMarked({ apiKey: API_KEY, fetch: deterministic }); + await expect(wm2.bulk(["https://a.example"])).rejects.toBeInstanceOf( + UnprocessableEntityError, + ); + // 4xx is deterministic — replaying just reproduces it. + expect(attempts4xx).toBe(1); + }); + + it("gives up after maxRetries and surfaces the connection error", async () => { + let attempts = 0; + const alwaysDown = (async () => { + attempts++; + throw new TypeError("network down"); + }) as typeof fetch; + + const wm = new WellMarked({ apiKey: API_KEY, fetch: alwaysDown, maxRetries: 1 }); + await expect(wm.bulk(["https://a.example"])).rejects.toBeInstanceOf(APIConnectionError); + expect(attempts).toBe(2); + }); + + it("maxRetries: 0 disables retrying entirely", async () => { + let attempts = 0; + const alwaysDown = (async () => { + attempts++; + throw new TypeError("network down"); + }) as typeof fetch; + + const wm = new WellMarked({ apiKey: API_KEY, fetch: alwaysDown, maxRetries: 0 }); + await expect(wm.bulk(["https://a.example"])).rejects.toBeInstanceOf(APIConnectionError); + expect(attempts).toBe(1); + }); + + it("a client-wide Idempotency-Key must NOT make /extract retryable", async () => { + // Regression: safeToReplay once read the MERGED headers. Idempotency-Key + // isn't reserved, so setHeader() could smuggle one onto every request — + // making POST /extract look replay-safe. The API bills /extract on + // arrival and ignores the header, so retrying it double-charges. + let attempts = 0; + const failingFetch = (async () => { + attempts++; + throw new TypeError("network down"); + }) as typeof fetch; + + const wm = new WellMarked({ apiKey: API_KEY, fetch: failingFetch }); + wm.setHeader("Idempotency-Key", "smuggled"); + + await expect(wm.extract("https://a.example")).rejects.toBeInstanceOf(APIConnectionError); + expect(attempts).toBe(1); + }); + + it("a constructor-level Idempotency-Key must NOT make /extract retryable", async () => { + let attempts = 0; + const failingFetch = (async () => { + attempts++; + throw new TypeError("network down"); + }) as typeof fetch; + + const wm = new WellMarked({ + apiKey: API_KEY, + fetch: failingFetch, + headers: { "Idempotency-Key": "smuggled" }, + }); + + await expect(wm.extract("https://a.example")).rejects.toBeInstanceOf(APIConnectionError); + expect(attempts).toBe(1); + }); +}); + +// ── Phase 5 continuity: policy overrides, key CRUD, logs ───────────────────── + +describe("policy overrides", () => { + it("sends allow_domains / deny_patterns / respect_robots on extract", async () => { + mock.on("POST", "/extract", () => + jsonResponse(200, { + markdown: "# ok", + metadata: { url: "https://a.example" }, + request_id: "r1", + }), + ); + const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + await wm.extract("https://a.example", { + allowDomains: ["a.example"], + denyPatterns: ["*/admin/*"], + respectRobots: "strict", + }); + const sent = mock.calls.at(-1)!.body as Record; + expect(sent.allow_domains).toEqual(["a.example"]); + expect(sent.deny_patterns).toEqual(["*/admin/*"]); + expect(sent.respect_robots).toBe("strict"); + }); + + it("omits policy fields left unset (never clobbers the key's policy)", async () => { + mock.on("POST", "/extract", () => + jsonResponse(200, { + markdown: "# ok", + metadata: { url: "https://a.example" }, + request_id: "r1", + }), + ); + const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + await wm.extract("https://a.example"); + const sent = mock.calls.at(-1)!.body as Record; + expect("allow_domains" in sent).toBe(false); + expect("deny_patterns" in sent).toBe(false); + expect("respect_robots" in sent).toBe(false); + }); + + it("surfaces a policy denial on extract as PermissionDeniedError", async () => { + mock.on("POST", "/extract", () => + jsonResponse(403, { + error: { code: "domain_denied", message: "denied", retry: false }, + }), + ); + const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + await expect(wm.extract("https://blocked.example")).rejects.toMatchObject({ + code: "domain_denied", + }); + await expect(wm.extract("https://blocked.example")).rejects.toBeInstanceOf( + PermissionDeniedError, + ); + }); +}); + +describe("key management", () => { + it("createKey posts scopes + name and returns the raw key once", async () => { + mock.on("POST", "/keys", () => + jsonResponse(200, { + id: "k1", + api_key: "wm_" + "b".repeat(40), + name: "ci", + scopes: ["extract"], + created_at: "2026-07-17T00:00:00Z", + }), + ); + const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const key = await wm.createKey(["extract"], { name: "ci" }); + expect(key.apiKey.startsWith("wm_")).toBe(true); + expect(key.scopes).toEqual(["extract"]); + expect(mock.calls.at(-1)!.body).toEqual({ scopes: ["extract"], name: "ci" }); + }); + + it("listKeys returns metadata with an active flag", async () => { + mock.on("GET", "/keys", () => + jsonResponse(200, { + keys: [ + { id: "k1", name: "default", scopes: ["*"], created_at: "2026-07-01T00:00:00Z", revoked_at: null }, + { id: "k2", name: "ci", scopes: ["extract"], created_at: "2026-07-02T00:00:00Z", revoked_at: "2026-07-03T00:00:00Z" }, + ], + }), + ); + const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const keys = await wm.listKeys(); + expect(keys.map((k) => k.id)).toEqual(["k1", "k2"]); + expect(keys[0]!.active).toBe(true); + expect(keys[1]!.active).toBe(false); + }); + + it("revokeKey issues a DELETE and returns the revocation", async () => { + mock.on("DELETE", "/keys/k2", () => + jsonResponse(200, { id: "k2", revoked_at: "2026-07-03T00:00:00Z" }), + ); + const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const revoked = await wm.revokeKey("k2"); + expect(revoked.id).toBe("k2"); + expect(mock.calls.at(-1)!.method).toBe("DELETE"); + }); +}); + +describe("getLogs", () => { + it("passes limit/offset and parses policy_decision + key_id", async () => { + mock.on("GET", "/logs", () => + jsonResponse(200, { + logs: [ + { + id: "r1", + timestamp: "2026-07-17T00:00:00Z", + target_url: "https://a.example", + status_code: 403, + duration_ms: 3, + error_code: "domain_denied", + render_js: false, + key_id: "k1", + policy_decision: "domain_denied", + }, + ], + limit: 50, + offset: 0, + has_more: true, + }), + ); + const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const page = await wm.getLogs({ limit: 50, offset: 0 }); + expect(page.hasMore).toBe(true); + expect(page.logs[0]!.policyDecision).toBe("domain_denied"); + expect(page.logs[0]!.keyId).toBe("k1"); + expect(mock.calls.at(-1)!.url).toContain("limit=50"); + expect(mock.calls.at(-1)!.url).toContain("offset=0"); + }); +}); + +// ── Phase 6: self-registration ─────────────────────────────────────────────── + +describe("register (static)", () => { + it("posts email with no auth header and returns the account", async () => { + mock.on("POST", "/register", () => + jsonResponse(200, { + api_key: "wm_" + "d".repeat(40), + user_id: "u1", + plan: "free", + scopes: ["extract"], + }), + ); + const account = await WellMarked.register("agent@example.com", { + baseUrl: BASE_URL, + fetch: mock.fetch, + }); + expect(account.apiKey.startsWith("wm_")).toBe(true); + expect(account.plan).toBe("free"); + expect(account.scopes).toEqual(["extract"]); + const sent = mock.calls.at(-1)!; + expect("authorization" in sent.headers).toBe(false); + expect(sent.body).toEqual({ email: "agent@example.com" }); + }); + + it("throws RateLimitError on register_rate_limited", async () => { + mock.on("POST", "/register", () => + jsonResponse(429, { + error: { code: "register_rate_limited", message: "slow down", retry: true }, + }), + ); + await expect( + WellMarked.register("agent@example.com", { baseUrl: BASE_URL, fetch: mock.fetch }), + ).rejects.toMatchObject({ code: "register_rate_limited" }); + }); +}); + +// ── Search ───────────────────────────────────────────────────────────────── + +describe("search", () => { + const SEARCH_BODY = { + query: "typescript generics", + results: [ + { url: "https://a.test/1", status: "ok", title: "A", snippet: "s1", markdown: "# A" }, + { url: "https://b.test/2", status: "error", title: "B", snippet: "s2", error: "target_timeout" }, + ], + request_id: "33333333-3333-3333-3333-333333333333", + }; + + it("returns parsed results and sends the expected payload", async () => { + mock.on("POST", "/search", () => jsonResponse(200, SEARCH_BODY)); + + const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const res = await wm.search("typescript generics", { numResults: 2 }); + + // Request shape reached the server unchanged. + expect(mock.calls.at(-1)?.body).toEqual({ + query: "typescript generics", + num_results: 2, + render_js: false, + }); + + expect(res.query).toBe("typescript generics"); + expect(res.requestId).toBe("33333333-3333-3333-3333-333333333333"); + expect(res.results).toHaveLength(2); + const ok = res.results[0]!; + const err = res.results[1]!; + expect(ok.ok).toBe(true); + expect(ok.markdown).toBe("# A"); + expect(ok.title).toBe("A"); + // A failed page still carries the provider snippet + a stable error code. + expect(err.ok).toBe(false); + expect(err.error).toBe("target_timeout"); + expect(err.snippet).toBe("s2"); + }); + + it("defaults num_results to 5 when omitted", async () => { + mock.on("POST", "/search", () => jsonResponse(200, { ...SEARCH_BODY, results: [] })); + const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + await wm.search("q"); + expect((mock.calls.at(-1)?.body as { num_results: number }).num_results).toBe(5); + }); + + it("throws PermissionDeniedError on the Pro+ plan gate", async () => { + mock.on("POST", "/search", () => + jsonResponse(403, { error: { code: "plan_not_supported", message: "Pro+ only." } }), + ); + const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + await expect(wm.search("q")).rejects.toBeInstanceOf(PermissionDeniedError); + await expect(wm.search("q")).rejects.toMatchObject({ code: "plan_not_supported" }); + }); +}); From 8343d25acadf2255b0d2300b19defa9eec201180 Mon Sep 17 00:00:00 2001 From: Grady Dillon Date: Mon, 20 Jul 2026 16:53:59 -0400 Subject: [PATCH 2/8] Fix TypeScript 6 build and typecheck failures caught by CI The repo pins typescript ^6.0.3, but a stale local node_modules had 5.9.3, so both of these passed locally and failed in CI: - lib ES2020 doesn't declare Array.prototype.at, which the tests use. Raised the type surface to ES2022 (emit stays at the ES2020 target); the package supports Node >=18, which has it. - The CJS build passes --moduleResolution node (node10), deprecated in TS 6 and a hard error. Silenced with ignoreDeprecations for now; moving the CJS output to node16 resolution is a separate change. Verified against the pinned toolchain (npm ci) this time: typecheck, test (66), and build all exit 0. Co-Authored-By: Claude Opus 4.8 --- tsconfig.build.json | 4 ++++ tsconfig.json | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/tsconfig.build.json b/tsconfig.build.json index 624bc7c..8ff3d7b 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -1,6 +1,10 @@ { "extends": "./tsconfig.json", "compilerOptions": { + // The CJS build passes --moduleResolution node (node10), which TS 6 + // deprecates and TS 7 will drop. Silence it for now; migrating the CJS + // output to node16 resolution is a separate, behaviour-affecting change. + "ignoreDeprecations": "6.0", "rootDir": "src", "declaration": true, "declarationMap": false, diff --git a/tsconfig.json b/tsconfig.json index e752a6d..edaf5b5 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,7 +1,10 @@ { "compilerOptions": { "target": "ES2020", - "lib": ["ES2020", "DOM"], + // ES2022 for the type surface only (emit stays at the ES2020 target): + // the package supports Node >=18, which has ES2022 array methods such as + // Array.prototype.at. Without this, TS 6 rejects `.at()` in the tests. + "lib": ["ES2022", "DOM"], "module": "ESNext", "moduleResolution": "Bundler", "strict": true, From 46c19fe6302a01d992a53659e00a6ecb61fe9d95 Mon Sep 17 00:00:00 2001 From: Grady Dillon Date: Mon, 20 Jul 2026 16:59:56 -0400 Subject: [PATCH 3/8] CI: skip the test runner on Node 18, which vitest 4 cannot start on vitest 4 imports node:util styleText, added in Node 20.12, so the runner dies at startup on 18. The library itself still supports Node >=18.17 per package.json engines, and that floor stays covered by install + typecheck + build; only the runner is skipped. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/ci.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aa80baf..c4c7109 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,5 +24,11 @@ jobs: cache: npm - run: npm ci - run: npm run typecheck - - run: npm test + # vitest 4 imports node:util's styleText, added in Node 20.12, so the + # runner cannot start on 18. The LIBRARY still supports Node >=18.17 + # (see package.json engines), and that floor stays covered here by + # install + typecheck + build; only the test runner is skipped. + - name: Test (vitest requires Node >= 20) + if: matrix.node != '18' + run: npm test - run: npm run build From 1e76749de5378e856811b44fd11dd199614e4462 Mon Sep 17 00:00:00 2001 From: Grady Dillon Date: Mon, 20 Jul 2026 19:22:38 -0400 Subject: [PATCH 4/8] Add `format` option and content types (Phase 4.5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit extract/bulk/crawl accept `format` (markdown | json | chunks | html | links). ExtractResult, BulkItem and CrawlItem now extend a shared `Content` interface carrying the five format-dependent fields plus `metrics`; exactly one is populated per result and `markdown` still holds the default. New exports: OutputFormat, Content, ContentBlock, Chunk, ContentMetrics, and `contentOf()` for reading whichever field came back. `ok` keys on `error` + contentOf rather than on `markdown` — the old form reported every successful links/html/chunks item as failed. The API's snake_case chunk offsets (start_token/end_token) map to camelCase (startToken/endToken); a test pins that contiguity survives the translation. typecheck, 70 tests and build all clean. Co-Authored-By: Claude Opus 4.8 --- src/client.ts | 28 ++++++++ src/index.ts | 6 ++ src/models.ts | 159 ++++++++++++++++++++++++++++++++++++++----- tests/client.test.ts | 131 ++++++++++++++++++++++++++++++++++- 4 files changed, 304 insertions(+), 20 deletions(-) diff --git a/src/client.ts b/src/client.ts index c634b1a..f19b97b 100644 --- a/src/client.ts +++ b/src/client.ts @@ -27,6 +27,7 @@ import { type CreatedKey, type ExtractResult, type LogsPage, + type OutputFormat, type RegisteredAccount, type RevokedKey, type RotatedKey, @@ -127,6 +128,14 @@ export interface ExtractOptions extends PolicyOverrideOptions { * Enterprise plan; Free returns `plan_not_supported`. */ renderJs?: boolean; + /** + * Output format. `"markdown"` (default), `"json"` (typed + * heading/paragraph/list/code blocks), `"chunks"` (contiguous 500-token + * windows for embedding), `"html"` (the raw fetched HTML) or `"links"` + * (every http(s) link found). The result populates the matching field; + * the others stay null. Use `contentOf(result)` to read whichever it is. + */ + format?: OutputFormat; } /** Options for `search`. */ @@ -222,12 +231,28 @@ export interface JobWebhookOptions { export interface BulkOptions extends JobWebhookOptions, PolicyOverrideOptions { renderJs?: boolean; + /** + * Output format. `"markdown"` (default), `"json"` (typed + * heading/paragraph/list/code blocks), `"chunks"` (contiguous 500-token + * windows for embedding), `"html"` (the raw fetched HTML) or `"links"` + * (every http(s) link found). The result populates the matching field; + * the others stay null. Use `contentOf(result)` to read whichever it is. + */ + format?: OutputFormat; } export interface CrawlOptions extends JobWebhookOptions, PolicyOverrideOptions { /** Max BFS depth from the root. Defaults to 1. Must be >= 0. */ depth?: number; renderJs?: boolean; + /** + * Output format. `"markdown"` (default), `"json"` (typed + * heading/paragraph/list/code blocks), `"chunks"` (contiguous 500-token + * windows for embedding), `"html"` (the raw fetched HTML) or `"links"` + * (every http(s) link found). The result populates the matching field; + * the others stay null. Use `contentOf(result)` to read whichever it is. + */ + format?: OutputFormat; } export interface WaitForJobOptions { @@ -436,6 +461,7 @@ export class WellMarked { const body = await this.request("POST", "/extract", { url, render_js: options.renderJs === true, + format: options.format ?? "markdown", ...policyOverrides(options), }); return extractResultFromResponse(body as Record); @@ -494,6 +520,7 @@ export class WellMarked { const payload: Record = { urls: urlList, render_js: options.renderJs === true, + format: options.format ?? "markdown", ...policyOverrides(options), }; if (options.webhookUrl !== undefined) { @@ -604,6 +631,7 @@ export class WellMarked { url, depth, render_js: options.renderJs === true, + format: options.format ?? "markdown", ...policyOverrides(options), }; if (options.webhookUrl !== undefined) { diff --git a/src/index.ts b/src/index.ts index 546fa15..db07c6b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -34,6 +34,10 @@ export { type ApiKeyInfo, type BulkItem, type BulkJob, + type Chunk, + type Content, + type ContentBlock, + type ContentMetrics, type CrawlItem, type CrawlJob, type CreatedKey, @@ -42,6 +46,7 @@ export { type JobStatus, type LogEntry, type LogsPage, + type OutputFormat, type RegisteredAccount, type RevokedKey, type RotatedKey, @@ -50,6 +55,7 @@ export { type SearchResults, type TruncatedReason, type Usage, + contentOf, isBulkJob, isCrawlJob, } from "./models.js"; diff --git a/src/models.ts b/src/models.ts index 3a76356..e31d9f7 100644 --- a/src/models.ts +++ b/src/models.ts @@ -44,9 +44,133 @@ export function extractionMetaFromDict(data: Record): Extractio }; } -/** Result of `POST /extract`. */ -export interface ExtractResult { - markdown: string; +// ── Output formats ─────────────────────────────────────────────────────────── + +/** Output format for `extract` / `bulk` / `crawl`. */ +export type OutputFormat = "markdown" | "json" | "chunks" | "html" | "links"; + +/** One typed block of a `format: "json"` document. */ +export interface ContentBlock { + type: "heading" | "paragraph" | "list" | "code"; + text: string; + /** 1-6 for headings; null for every other type. */ + level: number | null; +} + +/** + * One token window of a `format: "chunks"` document. + * + * Offsets index into the document's token stream and are contiguous — + * `chunks[i].endToken === chunks[i + 1].startToken` — so joining every `text` + * reproduces exactly what `format: "markdown"` would have returned. Windows are + * 500 tokens except where that would bisect a multi-byte character, in which + * case they run a token or two longer. + */ +export interface Chunk { + text: string; + startToken: number; + endToken: number; +} + +/** + * How much the extraction reduced the page, in tokens. + * + * `tokensSaved` is `inputTokens - outputTokens`: what you did NOT have to send + * a model versus feeding it the raw HTML. Every token field is null if the API + * could not load its tokenizer — the extraction still succeeds, the metrics + * just go unreported. + */ +export interface ContentMetrics { + contentBytes: number; + inputTokens: number | null; + outputTokens: number | null; + tokensSaved: number | null; + reductionPct: number | null; +} + +/** + * The format-dependent content fields shared by every extraction result. + * + * Exactly one of `markdown`/`blocks`/`chunks`/`html`/`links` is populated — + * whichever the request's `format` selected. `markdown` is the default, so code + * written before formats existed keeps working unchanged. + */ +export interface Content { + markdown: string | null; + blocks: ContentBlock[] | null; + chunks: Chunk[] | null; + html: string | null; + links: string[] | null; + metrics: ContentMetrics | null; +} + +function numOrNull(v: unknown): number | null { + return typeof v === "number" ? v : null; +} + +function contentFromDict(data: Record): Content { + const rawBlocks = data.blocks; + const rawChunks = data.chunks; + const rawMetrics = data.metrics; + const rawLinks = data.links; + + return { + markdown: typeof data.markdown === "string" ? data.markdown : null, + blocks: Array.isArray(rawBlocks) + ? rawBlocks.map((b) => { + const o = b as Record; + return { + type: (typeof o.type === "string" ? o.type : "paragraph") as ContentBlock["type"], + text: typeof o.text === "string" ? o.text : "", + level: numOrNull(o.level), + }; + }) + : null, + chunks: Array.isArray(rawChunks) + ? rawChunks.map((c) => { + const o = c as Record; + return { + text: typeof o.text === "string" ? o.text : "", + startToken: typeof o.start_token === "number" ? o.start_token : 0, + endToken: typeof o.end_token === "number" ? o.end_token : 0, + }; + }) + : null, + html: typeof data.html === "string" ? data.html : null, + links: Array.isArray(rawLinks) ? rawLinks.filter((l): l is string => typeof l === "string") : null, + metrics: + rawMetrics && typeof rawMetrics === "object" + ? (() => { + const m = rawMetrics as Record; + return { + contentBytes: typeof m.content_bytes === "number" ? m.content_bytes : 0, + inputTokens: numOrNull(m.input_tokens), + outputTokens: numOrNull(m.output_tokens), + tokensSaved: numOrNull(m.tokens_saved), + reductionPct: numOrNull(m.reduction_pct), + }; + })() + : null, + }; +} + +/** + * Whichever content field the response actually populated, or null. + * + * Handy when you don't care which format came back; read the specific field + * when you need a precise static type. + */ +export function contentOf(item: Content): string | ContentBlock[] | Chunk[] | string[] | null { + return item.markdown ?? item.blocks ?? item.chunks ?? item.html ?? item.links ?? null; +} + +/** + * Result of `POST /extract`. + * + * `markdown` is populated for the default `format: "markdown"`; other formats + * populate their own field instead (see `Content`). + */ +export interface ExtractResult extends Content { metadata: ExtractionMeta; requestId: string; } @@ -57,7 +181,7 @@ export function extractResultFromResponse(body: Record): Extrac ? (body.metadata as Record) : {}; return { - markdown: typeof body.markdown === "string" ? body.markdown : "", + ...contentFromDict(body), metadata: extractionMetaFromDict(rawMeta), requestId: typeof body.request_id === "string" ? body.request_id : "", }; @@ -139,35 +263,36 @@ export function searchResultsFromResponse(body: Record): Search * `domain_denied`, `internal_error`. Same convention as `CrawlItem.error`, so * results parse identically on either endpoint. */ -export interface BulkItem { +export interface BulkItem extends Content { url: string; - markdown: string | null; metadata: ExtractionMeta | null; error: string | null; - /** True when the item completed successfully (no error + has markdown). */ + /** True when the item completed successfully (no error + has content). */ readonly ok: boolean; } export function bulkItemFromDict(data: Record): BulkItem { const rawMeta = data.metadata; const url = typeof data.url === "string" ? data.url : ""; - const markdown = typeof data.markdown === "string" ? data.markdown : null; const error = typeof data.error === "string" ? data.error : null; const metadata = rawMeta && typeof rawMeta === "object" ? extractionMetaFromDict(rawMeta as Record) : null; // `ok` reads from `this` rather than a captured local so it tracks any - // post-construction mutation of `error` / `markdown`. Previously the + // post-construction mutation of `error` / the content fields. Previously the // getter closed over the initial values and lied about state if either // field was reassigned later. + // + // It keys on `error` + contentOf, NOT on `markdown`: a non-markdown format + // leaves markdown null on a perfectly successful item. return { + ...contentFromDict(data), url, - markdown, metadata, error, get ok(): boolean { - return this.error === null && this.markdown !== null; + return this.error === null && contentOf(this) !== null; }, }; } @@ -246,13 +371,12 @@ export function bulkJobFromResponse(body: Record): BulkJob { * Shape mirrors `BulkItem` with an added `depth` field showing how far * from the root URL this page sits in the BFS. */ -export interface CrawlItem { +export interface CrawlItem extends Content { url: string; depth: number; - markdown: string | null; metadata: ExtractionMeta | null; error: string | null; - /** True when the page completed successfully (no error + has markdown). */ + /** True when the page completed successfully (no error + has content). */ readonly ok: boolean; } @@ -260,21 +384,20 @@ export function crawlItemFromDict(data: Record): CrawlItem { const rawMeta = data.metadata; const url = typeof data.url === "string" ? data.url : ""; const depth = typeof data.depth === "number" ? data.depth : 0; - const markdown = typeof data.markdown === "string" ? data.markdown : null; const error = typeof data.error === "string" ? data.error : null; const metadata = rawMeta && typeof rawMeta === "object" ? extractionMetaFromDict(rawMeta as Record) : null; - // See BulkItem.ok — same `this`-based fix. + // See BulkItem.ok — same `this`-based getter, same content-not-markdown key. return { + ...contentFromDict(data), url, depth, - markdown, metadata, error, get ok(): boolean { - return this.error === null && this.markdown !== null; + return this.error === null && contentOf(this) !== null; }, }; } diff --git a/tests/client.test.ts b/tests/client.test.ts index 6db119a..61cad2b 100644 --- a/tests/client.test.ts +++ b/tests/client.test.ts @@ -14,8 +14,10 @@ import { UnprocessableEntityError, WellMarked, WellMarkedError, + contentOf, isCrawlJob, } from "../src/index.js"; +import { bulkItemFromDict } from "../src/models.js"; import { MockFetch, emptyResponse, jsonResponse } from "./helpers.js"; const API_KEY = "wm_" + "a".repeat(40); @@ -151,8 +153,20 @@ describe("extract", () => { const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); const result = await wm.extract("https://example.com"); expect(new Set(Object.keys(result))).toEqual( - new Set(["markdown", "metadata", "requestId"]), + new Set([ + "markdown", + "blocks", + "chunks", + "html", + "links", + "metrics", + "metadata", + "requestId", + ]), ); + // The default format still lands in `markdown`, unchanged. + expect(result.markdown).toBe("x"); + expect(result.blocks).toBeNull(); }); }); @@ -723,7 +737,11 @@ describe("request body", () => { const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); await wm.extract("https://example.com", { renderJs: true }); const call = mock.calls[0]!; - expect(call.body).toEqual({ url: "https://example.com", render_js: true }); + expect(call.body).toEqual({ + url: "https://example.com", + render_js: true, + format: "markdown", + }); }); it("sends url + depth + render_js on crawl", async () => { @@ -746,6 +764,7 @@ describe("request body", () => { url: "https://example.com", depth: 2, render_js: false, + format: "markdown", }); }); }); @@ -1184,3 +1203,111 @@ describe("search", () => { await expect(wm.search("q")).rejects.toMatchObject({ code: "plan_not_supported" }); }); }); + +// ── Output formats (Phase 4.5) ────────────────────────────────────────────── +// The format param must reach the wire, and each format's payload must land in +// its own field. Content silently arriving as null would look to the caller +// like a successful-but-empty extraction. + +describe("output formats", () => { + it("sends format and parses json blocks + metrics", async () => { + mock.on("POST", "/extract", () => + jsonResponse(200, { + blocks: [ + { type: "heading", text: "Title", level: 1 }, + { type: "paragraph", text: "Body text.", level: null }, + ], + metrics: { + content_bytes: 8000, + input_tokens: 2000, + output_tokens: 500, + tokens_saved: 1500, + reduction_pct: 75.0, + }, + metadata: { url: "https://example.com" }, + request_id: "id", + }), + ); + const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const result = await wm.extract("https://example.com", { format: "json" }); + + expect((mock.calls[0]!.body as Record).format).toBe("json"); + expect(result.markdown).toBeNull(); + expect(result.blocks?.map((b) => b.type)).toEqual(["heading", "paragraph"]); + expect(result.blocks?.[0]!.level).toBe(1); + expect(result.blocks?.[1]!.level).toBeNull(); + expect(result.metrics?.tokensSaved).toBe(1500); + expect(result.metrics?.reductionPct).toBe(75.0); + expect(contentOf(result)).toEqual(result.blocks); + }); + + it("parses chunks with contiguous snake_case offsets", async () => { + mock.on("POST", "/extract", () => + jsonResponse(200, { + chunks: [ + { text: "first ", start_token: 0, end_token: 500 }, + { text: "second", start_token: 500, end_token: 812 }, + ], + metadata: { url: "https://example.com" }, + request_id: "id", + }), + ); + const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const result = await wm.extract("https://example.com", { format: "chunks" }); + + expect(result.chunks?.map((c) => c.startToken)).toEqual([0, 500]); + // Contiguity survives the snake_case → camelCase mapping. + expect(result.chunks?.[0]!.endToken).toBe(result.chunks?.[1]!.startToken); + }); + + it("forwards format on bulk and crawl", async () => { + mock.on("POST", "/bulk", () => + jsonResponse(200, { + job_id: "j", + kind: "bulk", + status: "queued", + total: 1, + completed: 0, + results: [], + }), + ); + mock.on("POST", "/crawl", () => + jsonResponse(200, { + job_id: "c", + kind: "crawl", + status: "queued", + total: 0, + completed: 0, + truncated: false, + truncated_reason: null, + results: [], + }), + ); + const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + await wm.bulk(["https://a.test"], { format: "links" }); + await wm.crawl("https://b.test", { format: "html" }); + + expect((mock.calls[0]!.body as Record).format).toBe("links"); + expect((mock.calls[1]!.body as Record).format).toBe("html"); + }); + + it("marks a non-markdown bulk item as ok", () => { + // Negative control: keying `ok` on `markdown !== null` (as it did before + // formats existed) reports every successful links/html/chunks item failed. + const item = bulkItemFromDict({ + url: "https://a.test", + links: ["https://a.test/x"], + error: null, + }); + expect(item.ok).toBe(true); + expect(item.markdown).toBeNull(); + expect(contentOf(item)).toEqual(["https://a.test/x"]); + + const failed = bulkItemFromDict({ + url: "https://b.test", + error: "target_timeout", + }); + expect(failed.ok).toBe(false); + expect(contentOf(failed)).toBeNull(); + }); +}); From fc169db76874c3e231813e3eb482ff7284873e3b Mon Sep 17 00:00:00 2001 From: Grady Dillon Date: Wed, 22 Jul 2026 12:39:05 -0400 Subject: [PATCH 5/8] =?UTF-8?q?Remove=20baseUrl=20and=20fetch=20options=20?= =?UTF-8?q?=E2=80=94=20global=20fetch,=20fixed=20host?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Product decision: users get no capability to point WellMarked anywhere but the hosted API. baseUrl leaves both option interfaces, and the injected fetch goes with it (a custom fetch is the same capability by another door). The client now resolves globalThis.fetch lazily at call time — the constructor can't fail in odd environments, and test runners stub the global exactly as the runtime resolves it. Tests migrate from the fetch option to vi.stubGlobal("fetch", ...), plus a negative control: a baseUrl smuggled past the types with a cast is ignored and the request still hits api.wellmarked.io. BREAKING vs the published package: both options were public. Co-Authored-By: Claude Opus 4.8 --- README.md | 5 +- src/client.ts | 47 ++++++------- tests/client.test.ts | 154 +++++++++++++++++++++++++------------------ 3 files changed, 113 insertions(+), 93 deletions(-) diff --git a/README.md b/README.md index b5b294e..9baaf6c 100644 --- a/README.md +++ b/README.md @@ -298,14 +298,13 @@ All inherit from `WellMarkedError`. ```typescript new WellMarked({ apiKey: "wm_...", // or set WELLMARKED_API_KEY - baseUrl: "https://api.wellmarked.io", timeoutMs: 30_000, // per request, default 30s - fetch: customFetch, // optional: bring your own fetch + maxRetries: 2, // retries for safely replayable requests headers: { "X-Trace-Id": "..." }, // optional: extra headers on every request }); ``` -Passing your own `fetch` is useful for custom proxies, polyfills (e.g. `undici` with a custom dispatcher), or test mocking. Any function with the standard `fetch` signature works. +The client always talks to `https://api.wellmarked.io` using the global `fetch` (Node 18+ or any browser). ## TypeScript diff --git a/src/client.ts b/src/client.ts index f19b97b..74fda03 100644 --- a/src/client.ts +++ b/src/client.ts @@ -67,8 +67,6 @@ export interface WellMarkedOptions { * `WELLMARKED_API_KEY` environment variable (Node.js only). */ apiKey?: string; - /** API base URL. Override for testing. */ - baseUrl?: string; /** * Timeout for a single attempt, milliseconds. Defaults to 30000 (30s). * @@ -92,11 +90,6 @@ export interface WellMarkedOptions { * job rather than enqueuing a second one. */ maxRetries?: number; - /** - * Bring your own `fetch`. Defaults to the global `fetch`. Useful for - * polyfills, custom agents/proxies, or test mocking. - */ - fetch?: typeof fetch; /** * Extra headers sent on every request — useful for adding an internal * correlation id, a custom user agent suffix, etc. @@ -157,12 +150,8 @@ export interface CreateKeyOptions { /** Options for the static `WellMarked.register`. */ export interface RegisterOptions { - /** API base URL. Override for testing. */ - baseUrl?: string; /** Timeout for the single request, milliseconds. Defaults to 30000. */ timeoutMs?: number; - /** Bring your own `fetch`. Defaults to the global `fetch`. */ - fetch?: typeof fetch; } /** Options for `getLogs`. */ @@ -335,23 +324,13 @@ export class WellMarked { private readonly baseUrl: string; private readonly timeoutMs: number; private readonly maxRetries: number; - private readonly fetchImpl: typeof fetch; private readonly extraHeaders: Record; constructor(options: WellMarkedOptions = {}) { this.apiKey = resolveApiKey(options.apiKey); - this.baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, ""); + this.baseUrl = DEFAULT_BASE_URL; this.timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; this.maxRetries = Math.max(0, options.maxRetries ?? DEFAULT_MAX_RETRIES); - const f = options.fetch ?? (typeof fetch !== "undefined" ? fetch : undefined); - if (!f) { - throw new Error( - "No fetch implementation available. Pass `fetch:` to the client " + - "(undici, node-fetch, etc.) or upgrade to Node 18+.", - ); - } - // Bind so `this` isn't lost when calling globalThis.fetch. - this.fetchImpl = f.bind(globalThis) as typeof fetch; this.extraHeaders = {}; if (options.headers) { for (const [k, v] of Object.entries(options.headers)) { @@ -361,6 +340,23 @@ export class WellMarked { } } + /** + * The global `fetch`, resolved at call time (not captured at construction). + * Node 18+ and every browser provide it. Resolving lazily keeps the + * constructor infallible in odd environments and lets test runners stub + * `globalThis.fetch` whenever they like. + */ + private get fetchImpl(): typeof fetch { + const f = typeof fetch !== "undefined" ? fetch : undefined; + if (!f) { + throw new Error( + "No global fetch available. Node 18+ (or a browser) is required.", + ); + } + // Bind so `this` isn't lost when calling globalThis.fetch. + return f.bind(globalThis) as typeof fetch; + } + // ── Self-registration ─────────────────────────────────────────────────────── /** @@ -385,13 +381,12 @@ export class WellMarked { email: string, options: RegisterOptions = {}, ): Promise { - const baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, ""); + const baseUrl = DEFAULT_BASE_URL; const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; - const f = options.fetch ?? (typeof fetch !== "undefined" ? fetch : undefined); + const f = typeof fetch !== "undefined" ? fetch : undefined; if (!f) { throw new Error( - "No fetch implementation available. Pass `fetch:` to register() " + - "(undici, node-fetch, etc.) or upgrade to Node 18+.", + "No global fetch available. Node 18+ (or a browser) is required.", ); } const fetchImpl = f.bind(globalThis) as typeof fetch; diff --git a/tests/client.test.ts b/tests/client.test.ts index 61cad2b..8821c08 100644 --- a/tests/client.test.ts +++ b/tests/client.test.ts @@ -25,12 +25,17 @@ const BASE_URL = "https://api.wellmarked.io"; let mock: MockFetch; +// The client has no `fetch` option (and no `baseUrl`) — it always uses the +// global fetch against https://api.wellmarked.io. Tests therefore stub the +// GLOBAL, exactly as the runtime resolves it (lazily, at call time). beforeEach(() => { mock = new MockFetch(); + vi.stubGlobal("fetch", mock.fetch); }); afterEach(() => { mock.reset(); + vi.unstubAllGlobals(); }); // ── Extract ──────────────────────────────────────────────────────────────── @@ -51,7 +56,7 @@ describe("extract", () => { }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); const result = await wm.extract("https://example.com"); expect(result.markdown).toBe("## Hello"); @@ -74,7 +79,7 @@ describe("extract", () => { }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); await expect(wm.extract("https://example.com")).rejects.toMatchObject({ code: "rate_limit_exceeded", retryAfter: 1209600, @@ -104,7 +109,7 @@ describe("extract", () => { ), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); await expect(wm.extract("https://example.com")).rejects.toMatchObject({ code: "rate_limit_too_fast", retryAfter: 1, @@ -120,7 +125,7 @@ describe("extract", () => { }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); await expect(wm.extract("https://example.com")).rejects.toBeInstanceOf(AuthenticationError); }); @@ -131,7 +136,7 @@ describe("extract", () => { }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); let caught: unknown; try { await wm.extract("https://example.com"); @@ -150,7 +155,7 @@ describe("extract", () => { request_id: "id", }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); const result = await wm.extract("https://example.com"); expect(new Set(Object.keys(result))).toEqual( new Set([ @@ -168,6 +173,23 @@ describe("extract", () => { expect(result.markdown).toBe("x"); expect(result.blocks).toBeNull(); }); + + it("has no baseUrl escape hatch — a smuggled option is ignored", async () => { + // WellMarked only serves api.wellmarked.io. There is deliberately no + // baseUrl (or fetch) option; TypeScript rejects them at compile time, and + // a caller who force-casts past the types still ends up at the real API. + mock.on("POST", "/extract", () => + jsonResponse(200, { + markdown: "x", + metadata: { url: "https://example.com" }, + request_id: "id", + }), + ); + const smuggled = { apiKey: API_KEY, baseUrl: "http://localhost:8000" }; + const wm = new WellMarked(smuggled as ConstructorParameters[0]); + await wm.extract("https://example.com"); + expect(mock.calls.at(-1)!.url.startsWith(BASE_URL)).toBe(true); + }); }); // ── Bulk ─────────────────────────────────────────────────────────────────── @@ -184,7 +206,7 @@ describe("bulk", () => { }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); const job = await wm.bulk(["https://a.example", "https://b.example"]); expect(job.status).toBe("queued"); @@ -199,12 +221,12 @@ describe("bulk", () => { }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); await expect(wm.bulk(["https://a.example"])).rejects.toBeInstanceOf(PermissionDeniedError); }); it("rejects empty URL lists client-side with a clear error", async () => { - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); await expect(wm.bulk([])).rejects.toThrow(/at least one URL/); // No network call should have happened. expect(mock.calls.length).toBe(0); @@ -225,7 +247,7 @@ describe("getUsage", () => { }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); const usage = await wm.getUsage(); expect(usage.plan).toBe("pro"); @@ -257,7 +279,7 @@ describe("rotateKey", () => { }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); const rotated = await wm.rotateKey(); // Subsequent requests should carry the new bearer token. await wm.getUsage(); @@ -276,7 +298,7 @@ describe("api key resolution", () => { const original = process.env.WELLMARKED_API_KEY; delete process.env.WELLMARKED_API_KEY; try { - expect(() => new WellMarked({ fetch: mock.fetch })).toThrow(/No API key/); + expect(() => new WellMarked()).toThrow(/No API key/); } finally { if (original !== undefined) process.env.WELLMARKED_API_KEY = original; } @@ -286,7 +308,7 @@ describe("api key resolution", () => { const original = process.env.WELLMARKED_API_KEY; process.env.WELLMARKED_API_KEY = API_KEY; try { - const wm = new WellMarked({ fetch: mock.fetch }); + const wm = new WellMarked(); expect(wm._getApiKey()).toBe(API_KEY); } finally { if (original === undefined) delete process.env.WELLMARKED_API_KEY; @@ -312,7 +334,7 @@ describe("ExtractionMeta", () => { request_id: "id", }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); const result = await wm.extract("https://example.com"); const meta = result.metadata; expect(meta.url).toBe("https://example.com"); @@ -338,7 +360,7 @@ describe("ExtractionMeta", () => { request_id: "id", }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); const result = await wm.extract("https://example.com"); expect(result.metadata.title).toBeNull(); expect(result.metadata.author).toBeNull(); @@ -352,7 +374,7 @@ describe("ExtractionMeta", () => { describe("contract violations", () => { it("2xx with empty body raises a WellMarkedError", async () => { mock.on("POST", "/extract", () => emptyResponse(200)); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); await expect(wm.extract("https://example.com")).rejects.toThrow(/no JSON body/); await expect(wm.extract("https://example.com")).rejects.toBeInstanceOf(WellMarkedError); }); @@ -365,7 +387,8 @@ describe("transport errors", () => { const failingFetch: typeof fetch = async () => { throw new TypeError("fetch failed: ECONNREFUSED"); }; - const wm = new WellMarked({ apiKey: API_KEY, fetch: failingFetch }); + vi.stubGlobal("fetch", failingFetch); + const wm = new WellMarked({ apiKey: API_KEY }); await expect(wm.extract("https://example.com")).rejects.toBeInstanceOf(APIConnectionError); }); }); @@ -386,7 +409,7 @@ describe("crawl", () => { results: [], }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); const job = await wm.crawl("https://example.com", { depth: 2 }); expect(job.kind).toBe("crawl"); expect(job.status).toBe("queued"); @@ -401,7 +424,7 @@ describe("crawl", () => { error: { code: "plan_not_supported", message: "Upgrade." }, }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); await expect(wm.crawl("https://example.com", { depth: 1 })).rejects.toBeInstanceOf( PermissionDeniedError, ); @@ -413,7 +436,7 @@ describe("crawl", () => { error: { code: "crawl_depth_exceeded", message: "Pro caps at 5." }, }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); let caught: unknown; try { await wm.crawl("https://example.com", { depth: 10 }); @@ -425,7 +448,7 @@ describe("crawl", () => { }); it("rejects negative depth client-side", async () => { - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); await expect(wm.crawl("https://example.com", { depth: -1 })).rejects.toThrow( /depth must be >= 0/, ); @@ -455,7 +478,7 @@ describe("polymorphic getJob", () => { ], }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); const job = await wm.getJob(jobId); expect(job.kind).toBe("bulk"); expect(job.done).toBe(true); @@ -494,7 +517,7 @@ describe("polymorphic getJob", () => { ], }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); const job = await wm.getJob(jobId); expect(isCrawlJob(job)).toBe(true); if (isCrawlJob(job)) { @@ -544,7 +567,7 @@ describe("waitForJob", () => { }), ]); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); const job = await wm.waitForJob(jobId, { pollIntervalMs: 0, timeoutMs: 5000 }); expect(job.done).toBe(true); expect(job.completed).toBe(2); @@ -601,7 +624,7 @@ describe("waitForJob", () => { }), ]); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); const job = await wm.waitForJob(jobId, { pollIntervalMs: 0, timeoutMs: 5000 }); expect(isCrawlJob(job)).toBe(true); expect(job.done).toBe(true); @@ -624,7 +647,7 @@ describe("waitForJob", () => { results: [], }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); // Tight timeout — first iteration will check the deadline and bail. await expect( wm.waitForJob(jobId, { pollIntervalMs: 0, timeoutMs: 1 }), @@ -646,7 +669,6 @@ describe("custom headers", () => { const wm = new WellMarked({ apiKey: API_KEY, - fetch: mock.fetch, headers: { "X-Trace-Id": "abc123", "X-Tenant": "acme" }, }); await wm.extract("https://example.com"); @@ -667,7 +689,6 @@ describe("custom headers", () => { ); const wm = new WellMarked({ apiKey: API_KEY, - fetch: mock.fetch, headers: { Authorization: "Bearer wm_attacker", "X-Custom": "ok", @@ -687,7 +708,7 @@ describe("custom headers", () => { request_id: "id", }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); await wm.extract("https://example.com"); // no custom header yet wm.setHeader("X-Run-Id", "run-99"); @@ -716,7 +737,8 @@ describe("request timeout", () => { } }); - const wm = new WellMarked({ apiKey: API_KEY, fetch: slowFetch, timeoutMs: 50 }); + vi.stubGlobal("fetch", slowFetch); + const wm = new WellMarked({ apiKey: API_KEY, timeoutMs: 50 }); await expect(wm.extract("https://example.com")).rejects.toBeInstanceOf( APIConnectionError, ); @@ -734,7 +756,7 @@ describe("request body", () => { request_id: "id", }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); await wm.extract("https://example.com", { renderJs: true }); const call = mock.calls[0]!; expect(call.body).toEqual({ @@ -757,7 +779,7 @@ describe("request body", () => { results: [], }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); await wm.crawl("https://example.com", { depth: 2 }); const call = mock.calls[0]!; expect(call.body).toEqual({ @@ -785,7 +807,7 @@ const QUEUED_JOB = { describe("idempotency", () => { it("bulk() sends a generated Idempotency-Key when none is given", async () => { mock.on("POST", "/bulk", () => jsonResponse(200, QUEUED_JOB)); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); await wm.bulk(["https://a.example"]); @@ -794,7 +816,7 @@ describe("idempotency", () => { it("bulk() honours an explicit key", async () => { mock.on("POST", "/bulk", () => jsonResponse(200, QUEUED_JOB)); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); await wm.bulk(["https://a.example"], { idempotencyKey: "caller-chosen" }); @@ -805,7 +827,7 @@ describe("idempotency", () => { // Two submissions are two operations — sharing a key would make the // second replay the first one's job. mock.on("POST", "/bulk", () => jsonResponse(200, QUEUED_JOB)); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); await wm.bulk(["https://a.example"]); await wm.bulk(["https://b.example"]); @@ -819,7 +841,7 @@ describe("idempotency", () => { mock.on("POST", "/crawl", () => jsonResponse(200, { ...QUEUED_JOB, kind: "crawl", total: 0 }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); await wm.crawl("https://a.example"); @@ -831,7 +853,7 @@ describe("idempotency", () => { // The per-request key must win, or every later bulk() would replay the // first job. mock.on("POST", "/bulk", () => jsonResponse(200, QUEUED_JOB)); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); wm.setHeader("Idempotency-Key", "pinned-forever"); await wm.bulk(["https://a.example"]); @@ -845,7 +867,7 @@ describe("idempotency", () => { it("per-request headers still cannot override Authorization", async () => { mock.on("POST", "/bulk", () => jsonResponse(200, QUEUED_JOB)); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); await wm.bulk(["https://a.example"], { idempotencyKey: "k1" }); @@ -870,7 +892,8 @@ describe("retry", () => { return jsonResponse(200, QUEUED_JOB); }) as typeof fetch; - const wm = new WellMarked({ apiKey: API_KEY, fetch: failingFetch }); + vi.stubGlobal("fetch", failingFetch); + const wm = new WellMarked({ apiKey: API_KEY }); const job = await wm.bulk(["https://a.example"]); expect(attempts).toBe(2); @@ -889,7 +912,8 @@ describe("retry", () => { throw new TypeError("network down"); }) as typeof fetch; - const wm = new WellMarked({ apiKey: API_KEY, fetch: failingFetch }); + vi.stubGlobal("fetch", failingFetch); + const wm = new WellMarked({ apiKey: API_KEY }); await expect(wm.extract("https://a.example")).rejects.toBeInstanceOf(APIConnectionError); expect(attempts).toBe(1); }); @@ -901,7 +925,8 @@ describe("retry", () => { if (attempts === 1) return jsonResponse(503, { error: { code: "x", message: "down" } }); return jsonResponse(200, QUEUED_JOB); }) as typeof fetch; - const wm = new WellMarked({ apiKey: API_KEY, fetch: flaky }); + vi.stubGlobal("fetch", flaky); + const wm = new WellMarked({ apiKey: API_KEY }); await wm.bulk(["https://a.example"]); expect(attempts).toBe(2); @@ -912,7 +937,8 @@ describe("retry", () => { error: { code: "bulk_cap_exceeded", message: "too many" }, }); }) as typeof fetch; - const wm2 = new WellMarked({ apiKey: API_KEY, fetch: deterministic }); + vi.stubGlobal("fetch", deterministic); + const wm2 = new WellMarked({ apiKey: API_KEY }); await expect(wm2.bulk(["https://a.example"])).rejects.toBeInstanceOf( UnprocessableEntityError, ); @@ -927,7 +953,8 @@ describe("retry", () => { throw new TypeError("network down"); }) as typeof fetch; - const wm = new WellMarked({ apiKey: API_KEY, fetch: alwaysDown, maxRetries: 1 }); + vi.stubGlobal("fetch", alwaysDown); + const wm = new WellMarked({ apiKey: API_KEY, maxRetries: 1 }); await expect(wm.bulk(["https://a.example"])).rejects.toBeInstanceOf(APIConnectionError); expect(attempts).toBe(2); }); @@ -939,7 +966,8 @@ describe("retry", () => { throw new TypeError("network down"); }) as typeof fetch; - const wm = new WellMarked({ apiKey: API_KEY, fetch: alwaysDown, maxRetries: 0 }); + vi.stubGlobal("fetch", alwaysDown); + const wm = new WellMarked({ apiKey: API_KEY, maxRetries: 0 }); await expect(wm.bulk(["https://a.example"])).rejects.toBeInstanceOf(APIConnectionError); expect(attempts).toBe(1); }); @@ -955,7 +983,8 @@ describe("retry", () => { throw new TypeError("network down"); }) as typeof fetch; - const wm = new WellMarked({ apiKey: API_KEY, fetch: failingFetch }); + vi.stubGlobal("fetch", failingFetch); + const wm = new WellMarked({ apiKey: API_KEY }); wm.setHeader("Idempotency-Key", "smuggled"); await expect(wm.extract("https://a.example")).rejects.toBeInstanceOf(APIConnectionError); @@ -969,9 +998,9 @@ describe("retry", () => { throw new TypeError("network down"); }) as typeof fetch; + vi.stubGlobal("fetch", failingFetch); const wm = new WellMarked({ apiKey: API_KEY, - fetch: failingFetch, headers: { "Idempotency-Key": "smuggled" }, }); @@ -991,7 +1020,7 @@ describe("policy overrides", () => { request_id: "r1", }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); await wm.extract("https://a.example", { allowDomains: ["a.example"], denyPatterns: ["*/admin/*"], @@ -1011,7 +1040,7 @@ describe("policy overrides", () => { request_id: "r1", }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); await wm.extract("https://a.example"); const sent = mock.calls.at(-1)!.body as Record; expect("allow_domains" in sent).toBe(false); @@ -1025,7 +1054,7 @@ describe("policy overrides", () => { error: { code: "domain_denied", message: "denied", retry: false }, }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); await expect(wm.extract("https://blocked.example")).rejects.toMatchObject({ code: "domain_denied", }); @@ -1046,7 +1075,7 @@ describe("key management", () => { created_at: "2026-07-17T00:00:00Z", }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); const key = await wm.createKey(["extract"], { name: "ci" }); expect(key.apiKey.startsWith("wm_")).toBe(true); expect(key.scopes).toEqual(["extract"]); @@ -1062,7 +1091,7 @@ describe("key management", () => { ], }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); const keys = await wm.listKeys(); expect(keys.map((k) => k.id)).toEqual(["k1", "k2"]); expect(keys[0]!.active).toBe(true); @@ -1073,7 +1102,7 @@ describe("key management", () => { mock.on("DELETE", "/keys/k2", () => jsonResponse(200, { id: "k2", revoked_at: "2026-07-03T00:00:00Z" }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); const revoked = await wm.revokeKey("k2"); expect(revoked.id).toBe("k2"); expect(mock.calls.at(-1)!.method).toBe("DELETE"); @@ -1102,7 +1131,7 @@ describe("getLogs", () => { has_more: true, }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); const page = await wm.getLogs({ limit: 50, offset: 0 }); expect(page.hasMore).toBe(true); expect(page.logs[0]!.policyDecision).toBe("domain_denied"); @@ -1124,10 +1153,7 @@ describe("register (static)", () => { scopes: ["extract"], }), ); - const account = await WellMarked.register("agent@example.com", { - baseUrl: BASE_URL, - fetch: mock.fetch, - }); + const account = await WellMarked.register("agent@example.com"); expect(account.apiKey.startsWith("wm_")).toBe(true); expect(account.plan).toBe("free"); expect(account.scopes).toEqual(["extract"]); @@ -1143,7 +1169,7 @@ describe("register (static)", () => { }), ); await expect( - WellMarked.register("agent@example.com", { baseUrl: BASE_URL, fetch: mock.fetch }), + WellMarked.register("agent@example.com"), ).rejects.toMatchObject({ code: "register_rate_limited" }); }); }); @@ -1163,7 +1189,7 @@ describe("search", () => { it("returns parsed results and sends the expected payload", async () => { mock.on("POST", "/search", () => jsonResponse(200, SEARCH_BODY)); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); const res = await wm.search("typescript generics", { numResults: 2 }); // Request shape reached the server unchanged. @@ -1189,7 +1215,7 @@ describe("search", () => { it("defaults num_results to 5 when omitted", async () => { mock.on("POST", "/search", () => jsonResponse(200, { ...SEARCH_BODY, results: [] })); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); await wm.search("q"); expect((mock.calls.at(-1)?.body as { num_results: number }).num_results).toBe(5); }); @@ -1198,7 +1224,7 @@ describe("search", () => { mock.on("POST", "/search", () => jsonResponse(403, { error: { code: "plan_not_supported", message: "Pro+ only." } }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); await expect(wm.search("q")).rejects.toBeInstanceOf(PermissionDeniedError); await expect(wm.search("q")).rejects.toMatchObject({ code: "plan_not_supported" }); }); @@ -1228,7 +1254,7 @@ describe("output formats", () => { request_id: "id", }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); const result = await wm.extract("https://example.com", { format: "json" }); expect((mock.calls[0]!.body as Record).format).toBe("json"); @@ -1252,7 +1278,7 @@ describe("output formats", () => { request_id: "id", }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); const result = await wm.extract("https://example.com", { format: "chunks" }); expect(result.chunks?.map((c) => c.startToken)).toEqual([0, 500]); @@ -1283,7 +1309,7 @@ describe("output formats", () => { results: [], }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); await wm.bulk(["https://a.test"], { format: "links" }); await wm.crawl("https://b.test", { format: "html" }); From f4985b6436c07e56f180781c354f33495197eced Mon Sep 17 00:00:00 2001 From: Grady Dillon Date: Wed, 22 Jul 2026 14:49:43 -0400 Subject: [PATCH 6/8] search() takes the full extraction parameter set SearchOptions extends PolicyOverrideOptions and gains format; SearchResult extends Content so non-markdown formats land in their own fields (with metrics) via the shared contentFromDict path. Tool docs note the Pro+ gate on json/chunks. Co-Authored-By: Claude Opus 4.8 --- src/client.ts | 9 ++++++++- src/models.ts | 6 ++---- tests/client.test.ts | 35 ++++++++++++++++++++++++++++++++++- 3 files changed, 44 insertions(+), 6 deletions(-) diff --git a/src/client.ts b/src/client.ts index 74fda03..38e57e7 100644 --- a/src/client.ts +++ b/src/client.ts @@ -132,7 +132,7 @@ export interface ExtractOptions extends PolicyOverrideOptions { } /** Options for `search`. */ -export interface SearchOptions { +export interface SearchOptions extends PolicyOverrideOptions { /** * How many results to fetch + extract. Clamped to 1..10 server-side (a * search is one synchronous call). Default 5. @@ -140,6 +140,11 @@ export interface SearchOptions { numResults?: number; /** Render JS-heavy result pages before extracting. */ renderJs?: boolean; + /** + * Output format applied to every extracted result — search takes the full + * extraction parameter set, same as bulk and crawl. Default "markdown". + */ + format?: OutputFormat; } /** Options for `createKey`. */ @@ -483,6 +488,8 @@ export class WellMarked { query, num_results: options.numResults ?? 5, render_js: options.renderJs === true, + format: options.format ?? "markdown", + ...policyOverrides(options), }); return searchResultsFromResponse(body as Record); } diff --git a/src/models.ts b/src/models.ts index e31d9f7..6d04549 100644 --- a/src/models.ts +++ b/src/models.ts @@ -198,12 +198,11 @@ export function extractResultFromResponse(body: Record): Extrac * else the search provider's; `snippet` is always the provider's result * snippet, so a page that failed extraction still carries context. */ -export interface SearchResult { +export interface SearchResult extends Content { url: string; status: "ok" | "error"; title: string | null; snippet: string | null; - markdown: string | null; error: string | null; /** True when `status === "ok"`. */ readonly ok: boolean; @@ -214,14 +213,13 @@ export function searchResultFromDict(data: Record): SearchResul const status: "ok" | "error" = data.status === "ok" ? "ok" : "error"; const title = typeof data.title === "string" ? data.title : null; const snippet = typeof data.snippet === "string" ? data.snippet : null; - const markdown = typeof data.markdown === "string" ? data.markdown : null; const error = typeof data.error === "string" ? data.error : null; return { + ...contentFromDict(data), url, status, title, snippet, - markdown, error, get ok(): boolean { return this.status === "ok"; diff --git a/tests/client.test.ts b/tests/client.test.ts index 8821c08..b309a7c 100644 --- a/tests/client.test.ts +++ b/tests/client.test.ts @@ -1192,11 +1192,13 @@ describe("search", () => { const wm = new WellMarked({ apiKey: API_KEY }); const res = await wm.search("typescript generics", { numResults: 2 }); - // Request shape reached the server unchanged. + // Request shape reached the server unchanged (format defaults to markdown; + // policy overrides are omitted entirely when unset). expect(mock.calls.at(-1)?.body).toEqual({ query: "typescript generics", num_results: 2, render_js: false, + format: "markdown", }); expect(res.query).toBe("typescript generics"); @@ -1228,6 +1230,37 @@ describe("search", () => { await expect(wm.search("q")).rejects.toBeInstanceOf(PermissionDeniedError); await expect(wm.search("q")).rejects.toMatchObject({ code: "plan_not_supported" }); }); + + it("carries the full extraction parameter set: format + policy overrides", async () => { + mock.on("POST", "/search", () => + jsonResponse(200, { + query: "q", + results: [{ + url: "https://a.test/", status: "ok", snippet: "s", + chunks: [{ text: "hi", start_token: 0, end_token: 2 }], + }], + request_id: "33333333-3333-3333-3333-333333333333", + }), + ); + const wm = new WellMarked({ apiKey: API_KEY }); + const res = await wm.search("q", { + format: "chunks", + allowDomains: ["a.test"], + respectRobots: "strict", + }); + + const sent = mock.calls.at(-1)?.body as Record; + expect(sent.format).toBe("chunks"); + expect(sent.allow_domains).toEqual(["a.test"]); + expect(sent.respect_robots).toBe("strict"); + expect("deny_patterns" in sent).toBe(false); // unset overrides omitted + + const item = res.results[0]!; + expect(item.ok).toBe(true); + expect(item.markdown).toBeNull(); + expect(item.chunks?.[0]).toEqual({ text: "hi", startToken: 0, endToken: 2 }); + expect(contentOf(item)).toBe(item.chunks); // format-agnostic accessor + }); }); // ── Output formats (Phase 4.5) ────────────────────────────────────────────── From 61d0a6a9cd264d6c113bba982e23aba61e44a20e Mon Sep 17 00:00:00 2001 From: Grady Dillon Date: Wed, 22 Jul 2026 15:55:48 -0400 Subject: [PATCH 7/8] Add retry (extract/bulk/crawl) and crawl maxPages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit retry: server-side re-attempts on target_timeout, fresh connection per attempt, default 0, no upper bound. Not offered on search() — the API's 15s per-hit deadline can't absorb one. Distinct from the client's own maxRetries (transport retries between the SDK and the API). crawl({ maxPages: N }) narrows the plan's page cap (sent as max_pages); omitted = plan cap alone. Co-Authored-By: Claude Opus 4.8 --- README.md | 4 +++- src/client.ts | 27 +++++++++++++++++++++++++ tests/client.test.ts | 48 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9baaf6c..ee7d680 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,8 @@ for (const item of job.results) { } ``` +Pass `retry: N` to any of `extract`, `bulk`, or `crawl` to re-attempt timed-out fetches server-side (`target_timeout` only, fresh connection per attempt, default 0). On the synchronous `extract` each timed-out attempt takes 20–30s before the next fires, so aggressive values belong here on `bulk`, where the async workers absorb the wait. `search` doesn't take `retry` — its 15-second per-result deadline can't absorb one. + `getJob` and `waitForJob` are **polymorphic** — they work for both bulk and crawl `jobId`s. The SDK reads a `kind` discriminator from the API response and returns either a `BulkJob` or a `CrawlJob`. Use the `isCrawlJob(job)` type guard (or check `job.kind === "crawl"`) before reading crawl-specific fields like `job.truncated` or `item.depth`. ```typescript @@ -108,7 +110,7 @@ if (job.kind === "crawl" && job.truncated) { } ``` -Each successful page consumes one request from your monthly quota — failed pages (timeouts, robots-disallowed, no-content) are not billed. If you run out of quota mid-crawl the job finishes with `truncated: true`, `truncatedReason: "quota_exhausted"`. +Pass `maxPages: N` to stop the crawl after N successful pages — it can only narrow your plan's page cap, never widen it. Each successful page consumes one request from your monthly quota — failed pages (timeouts, robots-disallowed, no-content) are not billed. If you run out of quota mid-crawl the job finishes with `truncated: true`, `truncatedReason: "quota_exhausted"`. ## Webhooks diff --git a/src/client.ts b/src/client.ts index 38e57e7..dab6dba 100644 --- a/src/client.ts +++ b/src/client.ts @@ -129,6 +129,14 @@ export interface ExtractOptions extends PolicyOverrideOptions { * the others stay null. Use `contentOf(result)` to read whichever it is. */ format?: OutputFormat; + /** + * Server-side re-attempts on `target_timeout`, each on a fresh connection + * to the target. Default 0 (one attempt); no upper bound — but each + * timed-out attempt takes 20-30s on this synchronous call, so aggressive + * values belong on `bulk`. Distinct from the client's own `maxRetries` + * (transport retries between you and the API). + */ + retry?: number; } /** Options for `search`. */ @@ -233,6 +241,12 @@ export interface BulkOptions extends JobWebhookOptions, PolicyOverrideOptions { * the others stay null. Use `contentOf(result)` to read whichever it is. */ format?: OutputFormat; + /** + * Per-URL server-side re-attempts on `target_timeout` — see + * `ExtractOptions.retry`. The async workers absorb the retry time, so + * this is the natural home for aggressive values. + */ + retry?: number; } export interface CrawlOptions extends JobWebhookOptions, PolicyOverrideOptions { @@ -247,6 +261,13 @@ export interface CrawlOptions extends JobWebhookOptions, PolicyOverrideOptions { * the others stay null. Use `contentOf(result)` to read whichever it is. */ format?: OutputFormat; + /** Per-page server-side re-attempts on `target_timeout` — see `ExtractOptions.retry`. */ + retry?: number; + /** + * Stop the crawl after this many successful pages. Can only narrow your + * plan's page cap, never widen it. Must be >= 1. + */ + maxPages?: number; } export interface WaitForJobOptions { @@ -462,6 +483,7 @@ export class WellMarked { url, render_js: options.renderJs === true, format: options.format ?? "markdown", + retry: options.retry ?? 0, ...policyOverrides(options), }); return extractResultFromResponse(body as Record); @@ -523,6 +545,7 @@ export class WellMarked { urls: urlList, render_js: options.renderJs === true, format: options.format ?? "markdown", + retry: options.retry ?? 0, ...policyOverrides(options), }; if (options.webhookUrl !== undefined) { @@ -634,8 +657,12 @@ export class WellMarked { depth, render_js: options.renderJs === true, format: options.format ?? "markdown", + retry: options.retry ?? 0, ...policyOverrides(options), }; + if (options.maxPages !== undefined) { + payload.max_pages = options.maxPages; + } if (options.webhookUrl !== undefined) { payload.webhook_url = options.webhookUrl; payload.webhook_include_results = options.webhookIncludeResults === true; diff --git a/tests/client.test.ts b/tests/client.test.ts index b309a7c..4176ac0 100644 --- a/tests/client.test.ts +++ b/tests/client.test.ts @@ -763,6 +763,7 @@ describe("request body", () => { url: "https://example.com", render_js: true, format: "markdown", + retry: 0, }); }); @@ -787,8 +788,55 @@ describe("request body", () => { depth: 2, render_js: false, format: "markdown", + retry: 0, }); }); + + it("forwards retry on extract/bulk/crawl and maxPages on crawl", async () => { + // retry = server-side re-attempts on target_timeout. Search deliberately + // has no retry option (15s per-hit deadline), so its body must never + // carry one — pinned by the exact-body search test above. + mock.on("POST", "/extract", () => + jsonResponse(200, { + markdown: "## Hi", + metadata: { url: "https://example.com" }, + request_id: "id", + }), + ); + mock.on("POST", "/bulk", () => + jsonResponse(200, { + job_id: "x", status: "queued", total: 1, completed: 0, results: [], + }), + ); + mock.on("POST", "/crawl", () => + jsonResponse(200, { + job_id: "x", kind: "crawl", status: "queued", + total: 0, completed: 0, truncated: false, + truncated_reason: null, results: [], + }), + ); + const wm = new WellMarked({ apiKey: API_KEY }); + await wm.extract("https://example.com", { retry: 3 }); + await wm.bulk(["https://example.com"], { retry: 2 }); + await wm.crawl("https://example.com", { retry: 1, maxPages: 50 }); + + expect(mock.calls[0]!.body).toMatchObject({ retry: 3 }); + expect(mock.calls[1]!.body).toMatchObject({ retry: 2 }); + expect(mock.calls[2]!.body).toMatchObject({ retry: 1, max_pages: 50 }); + }); + + it("omits max_pages when maxPages is unset so the plan cap stands", async () => { + mock.on("POST", "/crawl", () => + jsonResponse(200, { + job_id: "x", kind: "crawl", status: "queued", + total: 0, completed: 0, truncated: false, + truncated_reason: null, results: [], + }), + ); + const wm = new WellMarked({ apiKey: API_KEY }); + await wm.crawl("https://example.com"); + expect(mock.calls[0]!.body).not.toHaveProperty("max_pages"); + }); }); // ── Idempotency-Key ──────────────────────────────────────────────────────── From ec5be1fd49d45f1caafb1e9852d977ac8dad60b4 Mon Sep 17 00:00:00 2001 From: Grady Dillon Date: Wed, 22 Jul 2026 17:10:49 -0400 Subject: [PATCH 8/8] Document the server-side retry lifetime ceiling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit retry's value stays uncapped, but the server stops re-attempting once a job's 6-hour lifetime is spent — state it where the option is documented so the JSDoc promise matches the API's behavior. Co-Authored-By: Claude Opus 4.8 --- src/client.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/client.ts b/src/client.ts index dab6dba..3c7291d 100644 --- a/src/client.ts +++ b/src/client.ts @@ -131,10 +131,11 @@ export interface ExtractOptions extends PolicyOverrideOptions { format?: OutputFormat; /** * Server-side re-attempts on `target_timeout`, each on a fresh connection - * to the target. Default 0 (one attempt); no upper bound — but each - * timed-out attempt takes 20-30s on this synchronous call, so aggressive - * values belong on `bulk`. Distinct from the client's own `maxRetries` - * (transport retries between you and the API). + * to the target. Default 0 (one attempt); no upper bound on the value, + * though the server stops re-attempting once a job's 6-hour lifetime is + * spent. Each timed-out attempt takes 20-30s on this synchronous call, so + * aggressive values belong on `bulk`. Distinct from the client's own + * `maxRetries` (transport retries between you and the API). */ retry?: number; }