diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aa80baf..c4c7109 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,5 +24,11 @@ jobs: cache: npm - run: npm ci - run: npm run typecheck - - run: npm test + # vitest 4 imports node:util's styleText, added in Node 20.12, so the + # runner cannot start on 18. The LIBRARY still supports Node >=18.17 + # (see package.json engines), and that floor stays covered here by + # install + typecheck + build; only the test runner is skipped. + - name: Test (vitest requires Node >= 20) + if: matrix.node != '18' + run: npm test - run: npm run build diff --git a/README.md b/README.md index a363153..ee7d680 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,8 @@ for (const item of job.results) { } ``` +Pass `retry: N` to any of `extract`, `bulk`, or `crawl` to re-attempt timed-out fetches server-side (`target_timeout` only, fresh connection per attempt, default 0). On the synchronous `extract` each timed-out attempt takes 20–30s before the next fires, so aggressive values belong here on `bulk`, where the async workers absorb the wait. `search` doesn't take `retry` — its 15-second per-result deadline can't absorb one. + `getJob` and `waitForJob` are **polymorphic** — they work for both bulk and crawl `jobId`s. The SDK reads a `kind` discriminator from the API response and returns either a `BulkJob` or a `CrawlJob`. Use the `isCrawlJob(job)` type guard (or check `job.kind === "crawl"`) before reading crawl-specific fields like `job.truncated` or `item.depth`. ```typescript @@ -108,7 +110,7 @@ if (job.kind === "crawl" && job.truncated) { } ``` -Each successful page consumes one request from your monthly quota — failed pages (timeouts, robots-disallowed, no-content) are not billed. If you run out of quota mid-crawl the job finishes with `truncated: true`, `truncatedReason: "quota_exhausted"`. +Pass `maxPages: N` to stop the crawl after N successful pages — it can only narrow your plan's page cap, never widen it. Each successful page consumes one request from your monthly quota — failed pages (timeouts, robots-disallowed, no-content) are not billed. If you run out of quota mid-crawl the job finishes with `truncated: true`, `truncatedReason: "quota_exhausted"`. ## Webhooks @@ -273,7 +275,7 @@ try { if (err instanceof RateLimitError) { console.log(`Quota hit. Resets in ${err.retryAfter}s.`); } else if (err instanceof UnprocessableEntityError) { - // err.code is one of: no_content, target_timeout, js_rendering_disabled, ... + // err.code is one of: no_content, target_timeout, ... console.log(`Extraction failed (${err.code}): ${err.message}`); } else { throw err; @@ -286,7 +288,7 @@ try { | `AuthenticationError` | 401 | `missing_api_key`, `invalid_api_key` | | `PermissionDeniedError` | 403 | `account_inactive`, `plan_not_supported`, `forbidden` | | `NotFoundError` | 404 | `job_not_found` | -| `UnprocessableEntityError` | 422 | `no_content`, `target_timeout`, `js_rendering_disabled`, `bulk_cap_exceeded`, `crawl_depth_exceeded` | +| `UnprocessableEntityError` | 422 | `no_content`, `target_timeout`, `bulk_cap_exceeded`, `crawl_depth_exceeded` | | `RateLimitError` | 429 | `rate_limit_too_fast` *(per-second cap; `retryAfterMs` carries the sub-second back-off)* · `rate_limit_exceeded` *(monthly quota; `retryAfter` in seconds)* | | `InternalServerError` | 5xx | — | | `APIConnectionError` | — | DNS / TCP / TLS / timeout failures, raised before any HTTP round-trip | @@ -298,14 +300,13 @@ All inherit from `WellMarkedError`. ```typescript new WellMarked({ apiKey: "wm_...", // or set WELLMARKED_API_KEY - baseUrl: "https://api.wellmarked.io", timeoutMs: 30_000, // per request, default 30s - fetch: customFetch, // optional: bring your own fetch + maxRetries: 2, // retries for safely replayable requests headers: { "X-Trace-Id": "..." }, // optional: extra headers on every request }); ``` -Passing your own `fetch` is useful for custom proxies, polyfills (e.g. `undici` with a custom dispatcher), or test mocking. Any function with the standard `fetch` signature works. +The client always talks to `https://api.wellmarked.io` using the global `fetch` (Node 18+ or any browser). ## TypeScript diff --git a/src/client.ts b/src/client.ts index 70077bc..3c7291d 100644 --- a/src/client.ts +++ b/src/client.ts @@ -21,23 +21,39 @@ import { fromResponse, } from "./errors.js"; import { + type ApiKeyInfo, type BulkJob, type CrawlJob, + type CreatedKey, type ExtractResult, + type LogsPage, + type OutputFormat, + type RegisteredAccount, + type RevokedKey, type RotatedKey, type RotatedWebhookSecret, + type SearchResults, type Usage, + apiKeyInfoFromDict, bulkJobFromResponse, crawlJobFromResponse, + createdKeyFromResponse, extractResultFromResponse, + logsPageFromResponse, + registeredAccountFromResponse, + revokedKeyFromResponse, rotatedKeyFromResponse, rotatedWebhookSecretFromResponse, + searchResultsFromResponse, usageFromResponse, } from "./models.js"; import { VERSION } from "./version.js"; const DEFAULT_BASE_URL = "https://api.wellmarked.io"; const DEFAULT_TIMEOUT_MS = 30_000; +// 3 attempts total. Enough to ride out a blip; low enough that a genuinely +// down API surfaces quickly instead of stalling the caller for a minute. +const DEFAULT_MAX_RETRIES = 2; const RESERVED_HEADERS = new Set([ "authorization", @@ -51,15 +67,29 @@ export interface WellMarkedOptions { * `WELLMARKED_API_KEY` environment variable (Node.js only). */ apiKey?: string; - /** API base URL. Override for testing. */ - baseUrl?: string; - /** Per-request timeout, milliseconds. Defaults to 30000 (30s). */ + /** + * Timeout for a single attempt, milliseconds. Defaults to 30000 (30s). + * + * This is **per attempt, not per call**. With the default `maxRetries: 2`, + * a retryable request can therefore take up to roughly + * `timeoutMs * 3` plus backoff (~92s at the defaults) before it gives up. + * Lower `maxRetries` if you need a tighter ceiling. + */ timeoutMs?: number; /** - * Bring your own `fetch`. Defaults to the global `fetch`. Useful for - * polyfills, custom agents/proxies, or test mocking. + * How many times to retry a request the SDK knows is safe to replay. + * Defaults to 2 (so 3 attempts total). Set 0 to disable. + * + * Only connection failures and 5xx responses are retried, and only for + * requests that can be replayed safely: GETs, and POSTs carrying an + * `Idempotency-Key` (which `bulk()` and `crawl()` send automatically). + * A `POST /extract` is never retried — the API bills it on arrival, and a + * connection error can't tell you whether it arrived. + * + * Retries reuse the same Idempotency-Key, so the API replays the original + * job rather than enqueuing a second one. */ - fetch?: typeof fetch; + maxRetries?: number; /** * Extra headers sent on every request — useful for adding an internal * correlation id, a custom user agent suffix, etc. @@ -70,12 +100,89 @@ export interface WellMarkedOptions { headers?: Record; } -export interface ExtractOptions { +/** + * Per-request compliance overrides, shared by extract / bulk / crawl. Each can + * only NARROW the API key's own policy server-side — add denies, restrict to a + * subset of the key's allowed domains, or upgrade robots to strict — never + * widen it. An omitted field leaves the key's policy untouched. + */ +export interface PolicyOverrideOptions { + /** Restrict this request to these domains (and their subdomains). */ + allowDomains?: string[]; + /** Extra deny globs for this request (matched on hostname and full URL). */ + denyPatterns?: string[]; + /** `"strict"` or `"lax"`; can tighten but not loosen the key's setting. */ + respectRobots?: "strict" | "lax"; +} + +export interface ExtractOptions extends PolicyOverrideOptions { /** - * Use Playwright to render JS-heavy pages. Requires a Pro/Enterprise - * plan AND `ENABLE_JS_RENDERING=true` on the API instance. + * Use Playwright to render JS-heavy pages. Requires a Pro, Growth, or + * Enterprise plan; Free returns `plan_not_supported`. */ renderJs?: boolean; + /** + * Output format. `"markdown"` (default), `"json"` (typed + * heading/paragraph/list/code blocks), `"chunks"` (contiguous 500-token + * windows for embedding), `"html"` (the raw fetched HTML) or `"links"` + * (every http(s) link found). The result populates the matching field; + * the others stay null. Use `contentOf(result)` to read whichever it is. + */ + format?: OutputFormat; + /** + * Server-side re-attempts on `target_timeout`, each on a fresh connection + * to the target. Default 0 (one attempt); no upper bound on the value, + * though the server stops re-attempting once a job's 6-hour lifetime is + * spent. Each timed-out attempt takes 20-30s on this synchronous call, so + * aggressive values belong on `bulk`. Distinct from the client's own + * `maxRetries` (transport retries between you and the API). + */ + retry?: number; +} + +/** Options for `search`. */ +export interface SearchOptions extends PolicyOverrideOptions { + /** + * How many results to fetch + extract. Clamped to 1..10 server-side (a + * search is one synchronous call). Default 5. + */ + numResults?: number; + /** Render JS-heavy result pages before extracting. */ + renderJs?: boolean; + /** + * Output format applied to every extracted result — search takes the full + * extraction parameter set, same as bulk and crawl. Default "markdown". + */ + format?: OutputFormat; +} + +/** Options for `createKey`. */ +export interface CreateKeyOptions { + /** A label for the key (shown in `listKeys`). Defaults to "default". */ + name?: string; +} + +/** Options for the static `WellMarked.register`. */ +export interface RegisterOptions { + /** Timeout for the single request, milliseconds. Defaults to 30000. */ + timeoutMs?: number; +} + +/** Options for `getLogs`. */ +export interface GetLogsOptions { + /** Page size, 1–200. Defaults to 50. */ + limit?: number; + /** Row offset for pagination. Defaults to 0. */ + offset?: number; +} + +/** Build the snake_case policy-override body fields, omitting unset ones. */ +function policyOverrides(o: PolicyOverrideOptions): Record { + const out: Record = {}; + if (o.allowDomains !== undefined) out.allow_domains = o.allowDomains; + if (o.denyPatterns !== undefined) out.deny_patterns = o.denyPatterns; + if (o.respectRobots !== undefined) out.respect_robots = o.respectRobots; + return out; } /** @@ -100,16 +207,68 @@ export interface JobWebhookOptions { * `results_url` pointing back at `getJob`. */ webhookIncludeResults?: boolean; + /** + * Sent as the `Idempotency-Key` header, so a replayed submission returns + * the ORIGINAL job instead of enqueuing a second one and charging your + * quota twice. + * + * A key is generated automatically and reused across the SDK's own internal + * retries (see `maxRetries`), which covers the common case of a connection + * blip mid-submission. + * + * **Pass your own key to survive anything the SDK can't retry for you** — + * a process crash, or your code catching the error and calling `bulk()` + * again. That second call mints a *new* key and gets a *second* job unless + * you reuse a stable one. + * + * Same key + same body replays the original job. Same key + a *different* + * body throws `UnprocessableEntityError` (`idempotency_key_reuse`) — use a + * fresh key per distinct submission. Records expire after 6 hours, matching + * the job TTL. + * + * Note the body must match byte-for-byte: reordering `urls` counts as a + * different request. + */ + idempotencyKey?: string; } -export interface BulkOptions extends JobWebhookOptions { +export interface BulkOptions extends JobWebhookOptions, PolicyOverrideOptions { renderJs?: boolean; + /** + * Output format. `"markdown"` (default), `"json"` (typed + * heading/paragraph/list/code blocks), `"chunks"` (contiguous 500-token + * windows for embedding), `"html"` (the raw fetched HTML) or `"links"` + * (every http(s) link found). The result populates the matching field; + * the others stay null. Use `contentOf(result)` to read whichever it is. + */ + format?: OutputFormat; + /** + * Per-URL server-side re-attempts on `target_timeout` — see + * `ExtractOptions.retry`. The async workers absorb the retry time, so + * this is the natural home for aggressive values. + */ + retry?: number; } -export interface CrawlOptions extends JobWebhookOptions { +export interface CrawlOptions extends JobWebhookOptions, PolicyOverrideOptions { /** Max BFS depth from the root. Defaults to 1. Must be >= 0. */ depth?: number; renderJs?: boolean; + /** + * Output format. `"markdown"` (default), `"json"` (typed + * heading/paragraph/list/code blocks), `"chunks"` (contiguous 500-token + * windows for embedding), `"html"` (the raw fetched HTML) or `"links"` + * (every http(s) link found). The result populates the matching field; + * the others stay null. Use `contentOf(result)` to read whichever it is. + */ + format?: OutputFormat; + /** Per-page server-side re-attempts on `target_timeout` — see `ExtractOptions.retry`. */ + retry?: number; + /** + * Stop the crawl after this many successful pages. Can only narrow your + * plan's page cap, never widen it. Must be >= 1. + */ + maxPages?: number; } export interface WaitForJobOptions { @@ -145,16 +304,37 @@ function defaultHeaders(apiKey: string): Record { function mergeHeaders( apiKey: string, extra: Record | undefined, + perRequest?: Record, ): Record { const out = defaultHeaders(apiKey); - if (!extra) return out; - for (const [k, v] of Object.entries(extra)) { - if (RESERVED_HEADERS.has(k.toLowerCase())) continue; - out[k] = v; + // Client-wide headers first, then per-request ones — a header passed to a + // single call wins over the same header set on the client. Both are filtered + // against RESERVED_HEADERS: a stray Authorization from either source would + // break rotateKey() mid-session (see RESERVED_HEADERS). + for (const source of [extra, perRequest]) { + if (!source) continue; + for (const [k, v] of Object.entries(source)) { + if (RESERVED_HEADERS.has(k.toLowerCase())) continue; + out[k] = v; + } } return out; } +/** + * Idempotency keys must be unique per logical operation, which is exactly why + * they can't ride on the client-wide `headers` option: that would pin every + * subsequent call to the same key. Hence the per-request channel above. + */ +function newIdempotencyKey(): string { + const g = globalThis as { crypto?: { randomUUID?: () => string } }; + if (typeof g.crypto?.randomUUID === "function") return g.crypto.randomUUID(); + // Node 18.17+ and every modern browser have crypto.randomUUID. This is a + // last resort for exotic runtimes; uniqueness only has to hold within one + // caller's retry window, not globally. + return `wm-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 12)}`; +} + function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } @@ -170,22 +350,14 @@ export class WellMarked { private apiKey: string; private readonly baseUrl: string; private readonly timeoutMs: number; - private readonly fetchImpl: typeof fetch; + private readonly maxRetries: number; private readonly extraHeaders: Record; constructor(options: WellMarkedOptions = {}) { this.apiKey = resolveApiKey(options.apiKey); - this.baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, ""); + this.baseUrl = DEFAULT_BASE_URL; this.timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; - const f = options.fetch ?? (typeof fetch !== "undefined" ? fetch : undefined); - if (!f) { - throw new Error( - "No fetch implementation available. Pass `fetch:` to the client " + - "(undici, node-fetch, etc.) or upgrade to Node 18+.", - ); - } - // Bind so `this` isn't lost when calling globalThis.fetch. - this.fetchImpl = f.bind(globalThis) as typeof fetch; + this.maxRetries = Math.max(0, options.maxRetries ?? DEFAULT_MAX_RETRIES); this.extraHeaders = {}; if (options.headers) { for (const [k, v] of Object.entries(options.headers)) { @@ -195,6 +367,106 @@ export class WellMarked { } } + /** + * The global `fetch`, resolved at call time (not captured at construction). + * Node 18+ and every browser provide it. Resolving lazily keeps the + * constructor infallible in odd environments and lets test runners stub + * `globalThis.fetch` whenever they like. + */ + private get fetchImpl(): typeof fetch { + const f = typeof fetch !== "undefined" ? fetch : undefined; + if (!f) { + throw new Error( + "No global fetch available. Node 18+ (or a browser) is required.", + ); + } + // Bind so `this` isn't lost when calling globalThis.fetch. + return f.bind(globalThis) as typeof fetch; + } + + // ── Self-registration ─────────────────────────────────────────────────────── + + /** + * Self-register for a free, extract-only API key — no existing key needed. + * + * The zero-to-first-call path for an agent that discovered WellMarked + * programmatically. Returns a `RegisteredAccount` whose `apiKey` is a weak + * credential (Free plan, `scopes: ["extract"]`); build a client with it: + * + * const account = await WellMarked.register("agent@example.com"); + * const wm = new WellMarked({ apiKey: account.apiKey }); + * await wm.extract("https://example.com"); + * + * Not retried — registration mints an account and isn't idempotent. + * + * Throws: + * - `RateLimitError` — `register_rate_limited`: too many from this IP. + * - `APIStatusError` — `email_taken` (409), or `service_unavailable` (503) + * if the limiter backend is momentarily down. + */ + static async register( + email: string, + options: RegisterOptions = {}, + ): Promise { + const baseUrl = DEFAULT_BASE_URL; + const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; + const f = typeof fetch !== "undefined" ? fetch : undefined; + if (!f) { + throw new Error( + "No global fetch available. Node 18+ (or a browser) is required.", + ); + } + const fetchImpl = f.bind(globalThis) as typeof fetch; + + let controller: AbortController | null = null; + let timer: ReturnType | null = null; + const init: RequestInitWithSignal = { + method: "POST", + headers: { "Content-Type": "application/json", Accept: "application/json" }, + body: JSON.stringify({ email }), + }; + if (timeoutMs > 0 && typeof AbortController !== "undefined") { + controller = new AbortController(); + init.signal = controller.signal; + timer = setTimeout(() => controller!.abort(), timeoutMs); + } + + let response: Response; + try { + response = await fetchImpl(`${baseUrl}/register`, init as RequestInit); + } catch (err) { + throw new APIConnectionError( + `Could not reach the WellMarked API: ${stringifyError(err)}`, + { cause: err }, + ); + } finally { + if (timer !== null) clearTimeout(timer); + } + + let bodyText = ""; + try { + bodyText = await response.text(); + } catch (err) { + throw new APIConnectionError( + `Could not read API response body: ${stringifyError(err)}`, + { cause: err }, + ); + } + let body: unknown = null; + if (bodyText.length > 0) { + try { + body = JSON.parse(bodyText); + } catch { + body = null; + } + } + const data = parseResponse(response.status, body, response.headers) as Record< + string, + unknown + >; + return registeredAccountFromResponse(data); + } + // ── Endpoints ────────────────────────────────────────────────────────────── /** @@ -202,27 +474,67 @@ export class WellMarked { * * Throws: * - `RateLimitError` — monthly plan limit reached. - * - `UnprocessableEntityError` — `no_content`, `target_timeout`, or - * `js_rendering_disabled`. + * - `PermissionDeniedError` — `plan_not_supported`: `renderJs=true` + * on the Free plan. + * - `UnprocessableEntityError` — `no_content` or `target_timeout`. * - `AuthenticationError` — missing or invalid API key. */ async extract(url: string, options: ExtractOptions = {}): Promise { const body = await this.request("POST", "/extract", { url, render_js: options.renderJs === true, + format: options.format ?? "markdown", + retry: options.retry ?? 0, + ...policyOverrides(options), }); return extractResultFromResponse(body as Record); } + /** + * Search the web and extract each result to Markdown. Synchronous — one + * round trip, no job to poll. + * + * The query runs against the search provider and the result pages are + * extracted concurrently, returned together with a per-page `status` (a slow + * or blocked page becomes an error item, never sinks the call). Costs + * `1 + results.length` requests — one for the query, one per page returned. + * + * Throws: + * - `PermissionDeniedError` — `plan_not_supported`: search requires a + * Pro, Growth, or Enterprise plan. + * - `RateLimitError` — would exceed remaining monthly quota. + * - `InternalServerError` — the provider is unconfigured or + * unreachable (`code === "service_unavailable"`). + */ + async search(query: string, options: SearchOptions = {}): Promise { + const body = await this.request("POST", "/search", { + query, + num_results: options.numResults ?? 5, + render_js: options.renderJs === true, + format: options.format ?? "markdown", + ...policyOverrides(options), + }); + return searchResultsFromResponse(body as Record); + } + /** * Submit a batch of URLs for concurrent extraction. * * Returns immediately with `status="queued"`. Poll with `getJob` or * block with `waitForJob` to collect results. * + * An `Idempotency-Key` is generated per call and reused across the SDK's + * internal retries, so a connection blip replays the original job rather + * than enqueuing a second one. To be safe against retries *your own code* + * makes, pass a stable `idempotencyKey` — see `BulkOptions.idempotencyKey`. + * * Throws: - * - `PermissionDeniedError` — `plan_not_supported` (Free tier). - * - `UnprocessableEntityError` — `bulk_cap_exceeded` (50 on Pro, 200 on Growth). + * - `PermissionDeniedError` — `plan_not_supported`: `renderJs=true` + * on the Free plan. + * - `UnprocessableEntityError` — `bulk_cap_exceeded` (5 on Free, 50 on + * Pro, 200 on Growth), or + * `idempotency_key_reuse` if the key was + * already used for a different body. * - `RateLimitError` — would exceed remaining monthly quota. */ async bulk(urls: Iterable, options: BulkOptions = {}): Promise { @@ -233,12 +545,17 @@ export class WellMarked { const payload: Record = { urls: urlList, render_js: options.renderJs === true, + format: options.format ?? "markdown", + retry: options.retry ?? 0, + ...policyOverrides(options), }; if (options.webhookUrl !== undefined) { payload.webhook_url = options.webhookUrl; payload.webhook_include_results = options.webhookIncludeResults === true; } - const body = await this.request("POST", "/bulk", payload); + const body = await this.request("POST", "/bulk", payload, { + "Idempotency-Key": options.idempotencyKey ?? newIdempotencyKey(), + }); return bulkJobFromResponse(body as Record); } @@ -340,19 +657,32 @@ export class WellMarked { url, depth, render_js: options.renderJs === true, + format: options.format ?? "markdown", + retry: options.retry ?? 0, + ...policyOverrides(options), }; + if (options.maxPages !== undefined) { + payload.max_pages = options.maxPages; + } if (options.webhookUrl !== undefined) { payload.webhook_url = options.webhookUrl; payload.webhook_include_results = options.webhookIncludeResults === true; } - const body = await this.request("POST", "/crawl", payload); + const body = await this.request("POST", "/crawl", payload, { + "Idempotency-Key": options.idempotencyKey ?? newIdempotencyKey(), + }); return crawlJobFromResponse(body as Record); } // ── Custom headers ───────────────────────────────────────────────────────── /** - * Add or replace a per-request header for the rest of this client's life. + * Add or replace a header for the rest of this client's life. + * + * Client-wide, so it is the wrong place for `Idempotency-Key`: that must be + * unique per submission, and pinning one here would make every later + * `bulk()` / `crawl()` replay the first job. Pass `idempotencyKey` to those + * methods instead (they generate one per call by default). * * Authorization / Content-Type / Accept are reserved — calls that try * to set those are silently ignored. To rotate the bearer token, use @@ -414,6 +744,72 @@ export class WellMarked { return rotatedWebhookSecretFromResponse(body as Record); } + // ── Key management (scoped keys) ───────────────────────────────────────────── + + /** + * Mint a new scoped API key on this account. + * + * `scopes` is a non-empty subset of `extract`, `bulk`, `crawl`, `keys`; the + * calling key can only grant scopes it holds. The raw key is in the returned + * `apiKey` — store it, it's shown once. Requires the `keys` scope. Does not + * count toward your quota. + * + * Throws: + * - `PermissionDeniedError` — `insufficient_scope`: your key lacks + * `keys`, or a scope it doesn't hold. + * - `UnprocessableEntityError` — `invalid_request`: empty/unknown scopes. + */ + async createKey(scopes: string[], options: CreateKeyOptions = {}): Promise { + const body = await this.request("POST", "/keys", { + scopes, + name: options.name ?? "default", + }); + return createdKeyFromResponse(body as Record); + } + + /** + * List this account's keys (metadata only — never the raw values), including + * revoked ones (`revokedAt` set). Requires the `keys` scope. Does not count + * toward your quota. + */ + async listKeys(): Promise { + const body = (await this.request("GET", "/keys")) as Record; + const raw = Array.isArray(body.keys) ? body.keys : []; + return raw + .filter((k): k is Record => k !== null && typeof k === "object") + .map(apiKeyInfoFromDict); + } + + /** + * Revoke a key by id — it stops authenticating immediately. Idempotent. + * Requires the `keys` scope. Does not count toward your quota. + * + * Throws `NotFoundError` (`key_not_found`) if no such key exists on this + * account. + */ + async revokeKey(keyId: string): Promise { + const body = await this.request("DELETE", `/keys/${keyId}`); + return revokedKeyFromResponse(body as Record); + } + + // ── Audit log ──────────────────────────────────────────────────────────────── + + /** + * Return this account's request history, newest first — the audit trail + * (which key ran each call and how its policy decided). Own rows only. + * Paginate via `offset += limit` while `hasMore` is true. Does not count + * toward your quota. + */ + async getLogs(options: GetLogsOptions = {}): Promise { + const limit = options.limit ?? 50; + const offset = options.offset ?? 0; + const body = await this.request( + "GET", + `/logs?limit=${encodeURIComponent(limit)}&offset=${encodeURIComponent(offset)}`, + ); + return logsPageFromResponse(body as Record); + } + /** * Internal: read the current API key. Exposed for tests. * Not part of the public, semver-stable surface. @@ -428,58 +824,113 @@ export class WellMarked { method: string, path: string, json?: unknown, + perRequestHeaders?: Record, ): Promise { const url = `${this.baseUrl}${path}`; - const headers = mergeHeaders(this.apiKey, this.extraHeaders); + const headers = mergeHeaders(this.apiKey, this.extraHeaders, perRequestHeaders); + + // Only replay a request when replaying it is actually safe. + // + // A connection error is ambiguous: the request may well have reached the + // API and been executed — we just never saw the response. Blindly retrying + // a POST /extract would extract (and bill) twice. GETs are naturally safe, + // and a POST is safe exactly when it carries an Idempotency-Key, because + // then the API replays the original job instead of creating a second one. + // + // This is what makes the auto-generated key on bulk()/crawl() worth + // anything: the key is reused across THESE attempts, so all of them + // collapse to one job. A caller who catches an error and calls bulk() + // again still mints a new key and gets a second job — that's why the + // docs tell you to pass your own key for retries across calls. + // + // Note this inspects `perRequestHeaders`, NOT the merged set. + // Idempotency-Key is not a RESERVED_HEADER, so a caller can legally + // `setHeader("Idempotency-Key", ...)` client-wide — and reading the merged + // headers would then mark POST /extract replay-safe and retry it, billing + // twice. Only a key this call actually attached counts. + const safeToReplay = + method.toUpperCase() === "GET" || + Object.keys(perRequestHeaders ?? {}).some( + (h) => h.toLowerCase() === "idempotency-key", + ); + const attempts = safeToReplay ? this.maxRetries + 1 : 1; - const init: RequestInitWithSignal = { method, headers }; - if (json !== undefined) { - init.body = JSON.stringify(json); - } + let lastError: unknown; + for (let attempt = 0; attempt < attempts; attempt++) { + if (attempt > 0) await sleep(backoffMs(attempt)); - let controller: AbortController | null = null; - let timer: ReturnType | null = null; - if (this.timeoutMs > 0 && typeof AbortController !== "undefined") { - controller = new AbortController(); - init.signal = controller.signal; - timer = setTimeout(() => controller!.abort(), this.timeoutMs); - } + const init: RequestInitWithSignal = { method, headers }; + if (json !== undefined) { + init.body = JSON.stringify(json); + } - let response: Response; - try { - response = await this.fetchImpl(url, init as RequestInit); - } catch (err) { - throw new APIConnectionError( - `Could not reach the WellMarked API: ${stringifyError(err)}`, - { cause: err }, - ); - } finally { - if (timer !== null) clearTimeout(timer); - } + // A fresh controller per attempt: reusing an aborted signal would make + // every retry fail instantly. + let controller: AbortController | null = null; + let timer: ReturnType | null = null; + if (this.timeoutMs > 0 && typeof AbortController !== "undefined") { + controller = new AbortController(); + init.signal = controller.signal; + timer = setTimeout(() => controller!.abort(), this.timeoutMs); + } - let bodyText = ""; - try { - bodyText = await response.text(); - } catch (err) { - throw new APIConnectionError( - `Could not read API response body: ${stringifyError(err)}`, - { cause: err }, - ); - } + let response: Response; + try { + response = await this.fetchImpl(url, init as RequestInit); + } catch (err) { + lastError = new APIConnectionError( + `Could not reach the WellMarked API: ${stringifyError(err)}`, + { cause: err }, + ); + continue; + } finally { + if (timer !== null) clearTimeout(timer); + } - let body: unknown = null; - if (bodyText.length > 0) { + // 5xx is the other ambiguous case — the API may have committed the job + // before failing. 4xx is deterministic: replaying reproduces it, so + // don't waste the caller's time. + if (response.status >= 500 && attempt < attempts - 1) { + lastError = undefined; + continue; + } + + let bodyText = ""; try { - body = JSON.parse(bodyText); - } catch { - body = null; + bodyText = await response.text(); + } catch (err) { + throw new APIConnectionError( + `Could not read API response body: ${stringifyError(err)}`, + { cause: err }, + ); + } + + let body: unknown = null; + if (bodyText.length > 0) { + try { + body = JSON.parse(bodyText); + } catch { + body = null; + } } + + return parseResponse(response.status, body, response.headers); } - return parseResponse(response.status, body, response.headers); + // Exhausted the retries on a connection error. A 5xx that ran out of + // attempts fell through to parseResponse above and threw there. + throw lastError ?? new APIConnectionError("Could not reach the WellMarked API."); } } +/** Exponential backoff with jitter. Jitter matters when an agent fans out — + * without it, N clients retry in lockstep and hit the API as a thundering + * herd at exactly the moment it is already struggling. */ +function backoffMs(attempt: number): number { + const base = Math.min(500 * 2 ** (attempt - 1), 4000); + return base + Math.random() * 250; +} + function parseResponse( statusCode: number, body: unknown, diff --git a/src/errors.ts b/src/errors.ts index fdd3c3d..78e6075 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -102,7 +102,6 @@ export class NotFoundError extends APIStatusError { * Common `code` values: * - `no_content` — could not identify main content on the page * - `target_timeout` — the target URL timed out - * - `js_rendering_disabled` — `renderJs=true` but the server has it off * - `bulk_cap_exceeded` — more URLs than the plan allows per request * - `crawl_depth_exceeded` — requested depth above the plan cap */ diff --git a/src/index.ts b/src/index.ts index 406ff49..db07c6b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -14,9 +14,14 @@ export { WellMarked, type WellMarkedOptions, type ExtractOptions, + type SearchOptions, type BulkOptions, type CrawlOptions, type JobWebhookOptions, + type PolicyOverrideOptions, + type CreateKeyOptions, + type GetLogsOptions, + type RegisterOptions, type WaitForJobOptions, } from "./client.js"; export { @@ -26,17 +31,31 @@ export { type WebhookPayload, } from "./webhooks.js"; export { + type ApiKeyInfo, type BulkItem, type BulkJob, + type Chunk, + type Content, + type ContentBlock, + type ContentMetrics, type CrawlItem, type CrawlJob, + type CreatedKey, type ExtractionMeta, type ExtractResult, type JobStatus, + type LogEntry, + type LogsPage, + type OutputFormat, + type RegisteredAccount, + type RevokedKey, type RotatedKey, type RotatedWebhookSecret, + type SearchResult, + type SearchResults, type TruncatedReason, type Usage, + contentOf, isBulkJob, isCrawlJob, } from "./models.js"; diff --git a/src/models.ts b/src/models.ts index 2808553..6d04549 100644 --- a/src/models.ts +++ b/src/models.ts @@ -44,9 +44,133 @@ export function extractionMetaFromDict(data: Record): Extractio }; } -/** Result of `POST /extract`. */ -export interface ExtractResult { - markdown: string; +// ── Output formats ─────────────────────────────────────────────────────────── + +/** Output format for `extract` / `bulk` / `crawl`. */ +export type OutputFormat = "markdown" | "json" | "chunks" | "html" | "links"; + +/** One typed block of a `format: "json"` document. */ +export interface ContentBlock { + type: "heading" | "paragraph" | "list" | "code"; + text: string; + /** 1-6 for headings; null for every other type. */ + level: number | null; +} + +/** + * One token window of a `format: "chunks"` document. + * + * Offsets index into the document's token stream and are contiguous — + * `chunks[i].endToken === chunks[i + 1].startToken` — so joining every `text` + * reproduces exactly what `format: "markdown"` would have returned. Windows are + * 500 tokens except where that would bisect a multi-byte character, in which + * case they run a token or two longer. + */ +export interface Chunk { + text: string; + startToken: number; + endToken: number; +} + +/** + * How much the extraction reduced the page, in tokens. + * + * `tokensSaved` is `inputTokens - outputTokens`: what you did NOT have to send + * a model versus feeding it the raw HTML. Every token field is null if the API + * could not load its tokenizer — the extraction still succeeds, the metrics + * just go unreported. + */ +export interface ContentMetrics { + contentBytes: number; + inputTokens: number | null; + outputTokens: number | null; + tokensSaved: number | null; + reductionPct: number | null; +} + +/** + * The format-dependent content fields shared by every extraction result. + * + * Exactly one of `markdown`/`blocks`/`chunks`/`html`/`links` is populated — + * whichever the request's `format` selected. `markdown` is the default, so code + * written before formats existed keeps working unchanged. + */ +export interface Content { + markdown: string | null; + blocks: ContentBlock[] | null; + chunks: Chunk[] | null; + html: string | null; + links: string[] | null; + metrics: ContentMetrics | null; +} + +function numOrNull(v: unknown): number | null { + return typeof v === "number" ? v : null; +} + +function contentFromDict(data: Record): Content { + const rawBlocks = data.blocks; + const rawChunks = data.chunks; + const rawMetrics = data.metrics; + const rawLinks = data.links; + + return { + markdown: typeof data.markdown === "string" ? data.markdown : null, + blocks: Array.isArray(rawBlocks) + ? rawBlocks.map((b) => { + const o = b as Record; + return { + type: (typeof o.type === "string" ? o.type : "paragraph") as ContentBlock["type"], + text: typeof o.text === "string" ? o.text : "", + level: numOrNull(o.level), + }; + }) + : null, + chunks: Array.isArray(rawChunks) + ? rawChunks.map((c) => { + const o = c as Record; + return { + text: typeof o.text === "string" ? o.text : "", + startToken: typeof o.start_token === "number" ? o.start_token : 0, + endToken: typeof o.end_token === "number" ? o.end_token : 0, + }; + }) + : null, + html: typeof data.html === "string" ? data.html : null, + links: Array.isArray(rawLinks) ? rawLinks.filter((l): l is string => typeof l === "string") : null, + metrics: + rawMetrics && typeof rawMetrics === "object" + ? (() => { + const m = rawMetrics as Record; + return { + contentBytes: typeof m.content_bytes === "number" ? m.content_bytes : 0, + inputTokens: numOrNull(m.input_tokens), + outputTokens: numOrNull(m.output_tokens), + tokensSaved: numOrNull(m.tokens_saved), + reductionPct: numOrNull(m.reduction_pct), + }; + })() + : null, + }; +} + +/** + * Whichever content field the response actually populated, or null. + * + * Handy when you don't care which format came back; read the specific field + * when you need a precise static type. + */ +export function contentOf(item: Content): string | ContentBlock[] | Chunk[] | string[] | null { + return item.markdown ?? item.blocks ?? item.chunks ?? item.html ?? item.links ?? null; +} + +/** + * Result of `POST /extract`. + * + * `markdown` is populated for the default `format: "markdown"`; other formats + * populate their own field instead (see `Content`). + */ +export interface ExtractResult extends Content { metadata: ExtractionMeta; requestId: string; } @@ -57,50 +181,116 @@ export function extractResultFromResponse(body: Record): Extrac ? (body.metadata as Record) : {}; return { - markdown: typeof body.markdown === "string" ? body.markdown : "", + ...contentFromDict(body), metadata: extractionMetaFromDict(rawMeta), requestId: typeof body.request_id === "string" ? body.request_id : "", }; } +// ── Search ─────────────────────────────────────────────────────────────────── + +/** + * One result in a `SearchResults`. + * + * On success `status === "ok"` and `markdown` is populated; on a per-page + * failure `status === "error"` and `error` carries a stable code (same + * convention as `BulkItem.error`). `title` is the extracted title on success, + * else the search provider's; `snippet` is always the provider's result + * snippet, so a page that failed extraction still carries context. + */ +export interface SearchResult extends Content { + url: string; + status: "ok" | "error"; + title: string | null; + snippet: string | null; + error: string | null; + /** True when `status === "ok"`. */ + readonly ok: boolean; +} + +export function searchResultFromDict(data: Record): SearchResult { + const url = typeof data.url === "string" ? data.url : ""; + const status: "ok" | "error" = data.status === "ok" ? "ok" : "error"; + const title = typeof data.title === "string" ? data.title : null; + const snippet = typeof data.snippet === "string" ? data.snippet : null; + const error = typeof data.error === "string" ? data.error : null; + return { + ...contentFromDict(data), + url, + status, + title, + snippet, + error, + get ok(): boolean { + return this.status === "ok"; + }, + }; +} + +/** + * Result of `POST /search` — the query plus the extracted result pages. + * Synchronous: unlike bulk/crawl there is no job to poll; `results` is already + * populated, with partial failures marked per item. + */ +export interface SearchResults { + query: string; + results: SearchResult[]; + requestId: string; +} + +export function searchResultsFromResponse(body: Record): SearchResults { + const rawResults = Array.isArray(body.results) ? body.results : []; + const results = rawResults + .filter((r): r is Record => r !== null && typeof r === "object") + .map(searchResultFromDict); + return { + query: typeof body.query === "string" ? body.query : "", + results, + requestId: typeof body.request_id === "string" ? body.request_id : "", + }; +} + // ── Bulk ───────────────────────────────────────────────────────────────────── /** * One entry in a bulk job's `results` list. * * On success, `markdown` and `metadata` are populated and `error` is null. - * On a per-URL failure, `markdown`/`metadata` are null and `error` carries - * an API error code (e.g. `target_timeout`). + * On a per-URL failure, `markdown`/`metadata` are null and `error` carries a + * stable API error *code* — never a human message — e.g. `target_timeout`, + * `domain_denied`, `internal_error`. Same convention as `CrawlItem.error`, so + * results parse identically on either endpoint. */ -export interface BulkItem { +export interface BulkItem extends Content { url: string; - markdown: string | null; metadata: ExtractionMeta | null; error: string | null; - /** True when the item completed successfully (no error + has markdown). */ + /** True when the item completed successfully (no error + has content). */ readonly ok: boolean; } export function bulkItemFromDict(data: Record): BulkItem { const rawMeta = data.metadata; const url = typeof data.url === "string" ? data.url : ""; - const markdown = typeof data.markdown === "string" ? data.markdown : null; const error = typeof data.error === "string" ? data.error : null; const metadata = rawMeta && typeof rawMeta === "object" ? extractionMetaFromDict(rawMeta as Record) : null; // `ok` reads from `this` rather than a captured local so it tracks any - // post-construction mutation of `error` / `markdown`. Previously the + // post-construction mutation of `error` / the content fields. Previously the // getter closed over the initial values and lied about state if either // field was reassigned later. + // + // It keys on `error` + contentOf, NOT on `markdown`: a non-markdown format + // leaves markdown null on a perfectly successful item. return { + ...contentFromDict(data), url, - markdown, metadata, error, get ok(): boolean { - return this.error === null && this.markdown !== null; + return this.error === null && contentOf(this) !== null; }, }; } @@ -179,13 +369,12 @@ export function bulkJobFromResponse(body: Record): BulkJob { * Shape mirrors `BulkItem` with an added `depth` field showing how far * from the root URL this page sits in the BFS. */ -export interface CrawlItem { +export interface CrawlItem extends Content { url: string; depth: number; - markdown: string | null; metadata: ExtractionMeta | null; error: string | null; - /** True when the page completed successfully (no error + has markdown). */ + /** True when the page completed successfully (no error + has content). */ readonly ok: boolean; } @@ -193,21 +382,20 @@ export function crawlItemFromDict(data: Record): CrawlItem { const rawMeta = data.metadata; const url = typeof data.url === "string" ? data.url : ""; const depth = typeof data.depth === "number" ? data.depth : 0; - const markdown = typeof data.markdown === "string" ? data.markdown : null; const error = typeof data.error === "string" ? data.error : null; const metadata = rawMeta && typeof rawMeta === "object" ? extractionMetaFromDict(rawMeta as Record) : null; - // See BulkItem.ok — same `this`-based fix. + // See BulkItem.ok — same `this`-based getter, same content-not-markdown key. return { + ...contentFromDict(data), url, depth, - markdown, metadata, error, get ok(): boolean { - return this.error === null && this.markdown !== null; + return this.error === null && contentOf(this) !== null; }, }; } @@ -356,6 +544,156 @@ export function rotatedWebhookSecretFromResponse( }; } +// ── Self-registration ──────────────────────────────────────────────────────── + +/** + * Result of `WellMarked.register` (`POST /register`). + * + * `apiKey` is the new raw key — shown once, store it. The account is + * deliberately weak: `plan === "free"` and `scopes === ["extract"]`. Build a + * client with it via `new WellMarked({ apiKey: account.apiKey })`. + */ +export interface RegisteredAccount { + apiKey: string; + userId: string; + plan: string; + scopes: string[]; +} + +export function registeredAccountFromResponse( + body: Record, +): RegisteredAccount { + return { + apiKey: typeof body.api_key === "string" ? body.api_key : "", + userId: typeof body.user_id === "string" ? body.user_id : "", + plan: typeof body.plan === "string" ? body.plan : "", + scopes: Array.isArray(body.scopes) ? (body.scopes as string[]) : [], + }; +} + +// ── Key management (scoped keys) ───────────────────────────────────────────── + +/** + * Result of `createKey` (`POST /keys`). `apiKey` is the new raw key — shown + * once, store it before discarding this object. `scopes` is the subset it was + * granted (extract / bulk / crawl / keys). + */ +export interface CreatedKey { + id: string; + apiKey: string; + name: string; + scopes: string[]; + createdAt: Date | null; +} + +export function createdKeyFromResponse(body: Record): CreatedKey { + return { + id: typeof body.id === "string" ? body.id : "", + apiKey: typeof body.api_key === "string" ? body.api_key : "", + name: typeof body.name === "string" ? body.name : "", + scopes: Array.isArray(body.scopes) ? (body.scopes as string[]) : [], + createdAt: parseDate(body.created_at), + }; +} + +/** + * One key's metadata from `listKeys` (`GET /keys`). Never carries the raw key. + * `revokedAt` is set once the key has been revoked; `active` is a convenience. + */ +export interface ApiKeyInfo { + id: string; + name: string; + scopes: string[]; + createdAt: Date | null; + revokedAt: Date | null; + readonly active: boolean; +} + +export function apiKeyInfoFromDict(data: Record): ApiKeyInfo { + return { + id: typeof data.id === "string" ? data.id : "", + name: typeof data.name === "string" ? data.name : "", + scopes: Array.isArray(data.scopes) ? (data.scopes as string[]) : [], + createdAt: parseDate(data.created_at), + revokedAt: parseDate(data.revoked_at), + get active(): boolean { + return this.revokedAt === null; + }, + }; +} + +/** Result of `revokeKey` (`DELETE /keys/{id}`). */ +export interface RevokedKey { + id: string; + revokedAt: Date | null; +} + +export function revokedKeyFromResponse(body: Record): RevokedKey { + return { + id: typeof body.id === "string" ? body.id : "", + revokedAt: parseDate(body.revoked_at), + }; +} + +// ── Audit log ──────────────────────────────────────────────────────────────── + +/** + * One row of your request history from `getLogs` (`GET /logs`). + * + * `policyDecision` records how the key's compliance policy decided: + * `"allowed"` | `"domain_not_allowed"` | `"domain_denied"` | + * `"robots_disallowed"`. `keyId` attributes the request to a key. + */ +export interface LogEntry { + id: string; + timestamp: Date | null; + targetUrl: string; + statusCode: number; + durationMs: number; + errorCode: string | null; + renderJs: boolean | null; + keyId: string | null; + policyDecision: string | null; +} + +export function logEntryFromDict(data: Record): LogEntry { + return { + id: typeof data.id === "string" ? data.id : "", + timestamp: parseDate(data.timestamp), + targetUrl: typeof data.target_url === "string" ? data.target_url : "", + statusCode: typeof data.status_code === "number" ? data.status_code : 0, + durationMs: typeof data.duration_ms === "number" ? data.duration_ms : 0, + errorCode: typeof data.error_code === "string" ? data.error_code : null, + renderJs: typeof data.render_js === "boolean" ? data.render_js : null, + keyId: typeof data.key_id === "string" ? data.key_id : null, + policyDecision: + typeof data.policy_decision === "string" ? data.policy_decision : null, + }; +} + +/** + * One page of `getLogs` results. `hasMore` is true when further rows exist + * beyond this page — advance by `offset += limit`. + */ +export interface LogsPage { + logs: LogEntry[]; + limit: number; + offset: number; + hasMore: boolean; +} + +export function logsPageFromResponse(body: Record): LogsPage { + const rawLogs = Array.isArray(body.logs) ? body.logs : []; + return { + logs: rawLogs + .filter((r): r is Record => r !== null && typeof r === "object") + .map(logEntryFromDict), + limit: typeof body.limit === "number" ? body.limit : 0, + offset: typeof body.offset === "number" ? body.offset : 0, + hasMore: body.has_more === true, + }; +} + // ── Type guards ────────────────────────────────────────────────────────────── export function isBulkJob(job: BulkJob | CrawlJob): job is BulkJob { diff --git a/tests/client.test.ts b/tests/client.test.ts index c80ec54..4176ac0 100644 --- a/tests/client.test.ts +++ b/tests/client.test.ts @@ -14,8 +14,10 @@ import { UnprocessableEntityError, WellMarked, WellMarkedError, + contentOf, isCrawlJob, } from "../src/index.js"; +import { bulkItemFromDict } from "../src/models.js"; import { MockFetch, emptyResponse, jsonResponse } from "./helpers.js"; const API_KEY = "wm_" + "a".repeat(40); @@ -23,12 +25,17 @@ const BASE_URL = "https://api.wellmarked.io"; let mock: MockFetch; +// The client has no `fetch` option (and no `baseUrl`) — it always uses the +// global fetch against https://api.wellmarked.io. Tests therefore stub the +// GLOBAL, exactly as the runtime resolves it (lazily, at call time). beforeEach(() => { mock = new MockFetch(); + vi.stubGlobal("fetch", mock.fetch); }); afterEach(() => { mock.reset(); + vi.unstubAllGlobals(); }); // ── Extract ──────────────────────────────────────────────────────────────── @@ -49,7 +56,7 @@ describe("extract", () => { }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); const result = await wm.extract("https://example.com"); expect(result.markdown).toBe("## Hello"); @@ -72,7 +79,7 @@ describe("extract", () => { }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); await expect(wm.extract("https://example.com")).rejects.toMatchObject({ code: "rate_limit_exceeded", retryAfter: 1209600, @@ -102,7 +109,7 @@ describe("extract", () => { ), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); await expect(wm.extract("https://example.com")).rejects.toMatchObject({ code: "rate_limit_too_fast", retryAfter: 1, @@ -118,7 +125,7 @@ describe("extract", () => { }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); await expect(wm.extract("https://example.com")).rejects.toBeInstanceOf(AuthenticationError); }); @@ -129,7 +136,7 @@ describe("extract", () => { }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); let caught: unknown; try { await wm.extract("https://example.com"); @@ -148,11 +155,40 @@ describe("extract", () => { request_id: "id", }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); const result = await wm.extract("https://example.com"); expect(new Set(Object.keys(result))).toEqual( - new Set(["markdown", "metadata", "requestId"]), + new Set([ + "markdown", + "blocks", + "chunks", + "html", + "links", + "metrics", + "metadata", + "requestId", + ]), ); + // The default format still lands in `markdown`, unchanged. + expect(result.markdown).toBe("x"); + expect(result.blocks).toBeNull(); + }); + + it("has no baseUrl escape hatch — a smuggled option is ignored", async () => { + // WellMarked only serves api.wellmarked.io. There is deliberately no + // baseUrl (or fetch) option; TypeScript rejects them at compile time, and + // a caller who force-casts past the types still ends up at the real API. + mock.on("POST", "/extract", () => + jsonResponse(200, { + markdown: "x", + metadata: { url: "https://example.com" }, + request_id: "id", + }), + ); + const smuggled = { apiKey: API_KEY, baseUrl: "http://localhost:8000" }; + const wm = new WellMarked(smuggled as ConstructorParameters[0]); + await wm.extract("https://example.com"); + expect(mock.calls.at(-1)!.url.startsWith(BASE_URL)).toBe(true); }); }); @@ -170,7 +206,7 @@ describe("bulk", () => { }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); const job = await wm.bulk(["https://a.example", "https://b.example"]); expect(job.status).toBe("queued"); @@ -185,12 +221,12 @@ describe("bulk", () => { }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); await expect(wm.bulk(["https://a.example"])).rejects.toBeInstanceOf(PermissionDeniedError); }); it("rejects empty URL lists client-side with a clear error", async () => { - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); await expect(wm.bulk([])).rejects.toThrow(/at least one URL/); // No network call should have happened. expect(mock.calls.length).toBe(0); @@ -211,7 +247,7 @@ describe("getUsage", () => { }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); const usage = await wm.getUsage(); expect(usage.plan).toBe("pro"); @@ -243,7 +279,7 @@ describe("rotateKey", () => { }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); const rotated = await wm.rotateKey(); // Subsequent requests should carry the new bearer token. await wm.getUsage(); @@ -262,7 +298,7 @@ describe("api key resolution", () => { const original = process.env.WELLMARKED_API_KEY; delete process.env.WELLMARKED_API_KEY; try { - expect(() => new WellMarked({ fetch: mock.fetch })).toThrow(/No API key/); + expect(() => new WellMarked()).toThrow(/No API key/); } finally { if (original !== undefined) process.env.WELLMARKED_API_KEY = original; } @@ -272,7 +308,7 @@ describe("api key resolution", () => { const original = process.env.WELLMARKED_API_KEY; process.env.WELLMARKED_API_KEY = API_KEY; try { - const wm = new WellMarked({ fetch: mock.fetch }); + const wm = new WellMarked(); expect(wm._getApiKey()).toBe(API_KEY); } finally { if (original === undefined) delete process.env.WELLMARKED_API_KEY; @@ -298,7 +334,7 @@ describe("ExtractionMeta", () => { request_id: "id", }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); const result = await wm.extract("https://example.com"); const meta = result.metadata; expect(meta.url).toBe("https://example.com"); @@ -324,7 +360,7 @@ describe("ExtractionMeta", () => { request_id: "id", }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); const result = await wm.extract("https://example.com"); expect(result.metadata.title).toBeNull(); expect(result.metadata.author).toBeNull(); @@ -338,7 +374,7 @@ describe("ExtractionMeta", () => { describe("contract violations", () => { it("2xx with empty body raises a WellMarkedError", async () => { mock.on("POST", "/extract", () => emptyResponse(200)); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); await expect(wm.extract("https://example.com")).rejects.toThrow(/no JSON body/); await expect(wm.extract("https://example.com")).rejects.toBeInstanceOf(WellMarkedError); }); @@ -351,7 +387,8 @@ describe("transport errors", () => { const failingFetch: typeof fetch = async () => { throw new TypeError("fetch failed: ECONNREFUSED"); }; - const wm = new WellMarked({ apiKey: API_KEY, fetch: failingFetch }); + vi.stubGlobal("fetch", failingFetch); + const wm = new WellMarked({ apiKey: API_KEY }); await expect(wm.extract("https://example.com")).rejects.toBeInstanceOf(APIConnectionError); }); }); @@ -372,7 +409,7 @@ describe("crawl", () => { results: [], }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); const job = await wm.crawl("https://example.com", { depth: 2 }); expect(job.kind).toBe("crawl"); expect(job.status).toBe("queued"); @@ -387,7 +424,7 @@ describe("crawl", () => { error: { code: "plan_not_supported", message: "Upgrade." }, }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); await expect(wm.crawl("https://example.com", { depth: 1 })).rejects.toBeInstanceOf( PermissionDeniedError, ); @@ -399,7 +436,7 @@ describe("crawl", () => { error: { code: "crawl_depth_exceeded", message: "Pro caps at 5." }, }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); let caught: unknown; try { await wm.crawl("https://example.com", { depth: 10 }); @@ -411,7 +448,7 @@ describe("crawl", () => { }); it("rejects negative depth client-side", async () => { - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); await expect(wm.crawl("https://example.com", { depth: -1 })).rejects.toThrow( /depth must be >= 0/, ); @@ -441,7 +478,7 @@ describe("polymorphic getJob", () => { ], }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); const job = await wm.getJob(jobId); expect(job.kind).toBe("bulk"); expect(job.done).toBe(true); @@ -480,7 +517,7 @@ describe("polymorphic getJob", () => { ], }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); const job = await wm.getJob(jobId); expect(isCrawlJob(job)).toBe(true); if (isCrawlJob(job)) { @@ -530,7 +567,7 @@ describe("waitForJob", () => { }), ]); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); const job = await wm.waitForJob(jobId, { pollIntervalMs: 0, timeoutMs: 5000 }); expect(job.done).toBe(true); expect(job.completed).toBe(2); @@ -587,7 +624,7 @@ describe("waitForJob", () => { }), ]); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); const job = await wm.waitForJob(jobId, { pollIntervalMs: 0, timeoutMs: 5000 }); expect(isCrawlJob(job)).toBe(true); expect(job.done).toBe(true); @@ -610,7 +647,7 @@ describe("waitForJob", () => { results: [], }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); // Tight timeout — first iteration will check the deadline and bail. await expect( wm.waitForJob(jobId, { pollIntervalMs: 0, timeoutMs: 1 }), @@ -632,7 +669,6 @@ describe("custom headers", () => { const wm = new WellMarked({ apiKey: API_KEY, - fetch: mock.fetch, headers: { "X-Trace-Id": "abc123", "X-Tenant": "acme" }, }); await wm.extract("https://example.com"); @@ -653,7 +689,6 @@ describe("custom headers", () => { ); const wm = new WellMarked({ apiKey: API_KEY, - fetch: mock.fetch, headers: { Authorization: "Bearer wm_attacker", "X-Custom": "ok", @@ -673,7 +708,7 @@ describe("custom headers", () => { request_id: "id", }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); await wm.extract("https://example.com"); // no custom header yet wm.setHeader("X-Run-Id", "run-99"); @@ -702,7 +737,8 @@ describe("request timeout", () => { } }); - const wm = new WellMarked({ apiKey: API_KEY, fetch: slowFetch, timeoutMs: 50 }); + vi.stubGlobal("fetch", slowFetch); + const wm = new WellMarked({ apiKey: API_KEY, timeoutMs: 50 }); await expect(wm.extract("https://example.com")).rejects.toBeInstanceOf( APIConnectionError, ); @@ -720,10 +756,15 @@ describe("request body", () => { request_id: "id", }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); await wm.extract("https://example.com", { renderJs: true }); const call = mock.calls[0]!; - expect(call.body).toEqual({ url: "https://example.com", render_js: true }); + expect(call.body).toEqual({ + url: "https://example.com", + render_js: true, + format: "markdown", + retry: 0, + }); }); it("sends url + depth + render_js on crawl", async () => { @@ -739,13 +780,641 @@ describe("request body", () => { results: [], }), ); - const wm = new WellMarked({ apiKey: API_KEY, fetch: mock.fetch }); + const wm = new WellMarked({ apiKey: API_KEY }); await wm.crawl("https://example.com", { depth: 2 }); const call = mock.calls[0]!; expect(call.body).toEqual({ url: "https://example.com", depth: 2, render_js: false, + format: "markdown", + retry: 0, + }); + }); + + it("forwards retry on extract/bulk/crawl and maxPages on crawl", async () => { + // retry = server-side re-attempts on target_timeout. Search deliberately + // has no retry option (15s per-hit deadline), so its body must never + // carry one — pinned by the exact-body search test above. + mock.on("POST", "/extract", () => + jsonResponse(200, { + markdown: "## Hi", + metadata: { url: "https://example.com" }, + request_id: "id", + }), + ); + mock.on("POST", "/bulk", () => + jsonResponse(200, { + job_id: "x", status: "queued", total: 1, completed: 0, results: [], + }), + ); + mock.on("POST", "/crawl", () => + jsonResponse(200, { + job_id: "x", kind: "crawl", status: "queued", + total: 0, completed: 0, truncated: false, + truncated_reason: null, results: [], + }), + ); + const wm = new WellMarked({ apiKey: API_KEY }); + await wm.extract("https://example.com", { retry: 3 }); + await wm.bulk(["https://example.com"], { retry: 2 }); + await wm.crawl("https://example.com", { retry: 1, maxPages: 50 }); + + expect(mock.calls[0]!.body).toMatchObject({ retry: 3 }); + expect(mock.calls[1]!.body).toMatchObject({ retry: 2 }); + expect(mock.calls[2]!.body).toMatchObject({ retry: 1, max_pages: 50 }); + }); + + it("omits max_pages when maxPages is unset so the plan cap stands", async () => { + mock.on("POST", "/crawl", () => + jsonResponse(200, { + job_id: "x", kind: "crawl", status: "queued", + total: 0, completed: 0, truncated: false, + truncated_reason: null, results: [], + }), + ); + const wm = new WellMarked({ apiKey: API_KEY }); + await wm.crawl("https://example.com"); + expect(mock.calls[0]!.body).not.toHaveProperty("max_pages"); + }); +}); + +// ── Idempotency-Key ──────────────────────────────────────────────────────── +// The header is what makes a retried /bulk replay the original job instead of +// double-charging the caller's quota. If the SDK silently stops sending it, +// the API's protection is inert and nothing else would catch it. + +const QUEUED_JOB = { + job_id: "1c4f9a02-0000-0000-0000-000000000000", + status: "queued", + total: 1, + completed: 0, + results: [], +}; + +describe("idempotency", () => { + it("bulk() sends a generated Idempotency-Key when none is given", async () => { + mock.on("POST", "/bulk", () => jsonResponse(200, QUEUED_JOB)); + const wm = new WellMarked({ apiKey: API_KEY }); + + await wm.bulk(["https://a.example"]); + + expect(mock.calls[0]!.headers["idempotency-key"]).toBeTruthy(); + }); + + it("bulk() honours an explicit key", async () => { + mock.on("POST", "/bulk", () => jsonResponse(200, QUEUED_JOB)); + const wm = new WellMarked({ apiKey: API_KEY }); + + await wm.bulk(["https://a.example"], { idempotencyKey: "caller-chosen" }); + + expect(mock.calls[0]!.headers["idempotency-key"]).toBe("caller-chosen"); + }); + + it("gives each submission a distinct generated key", async () => { + // Two submissions are two operations — sharing a key would make the + // second replay the first one's job. + mock.on("POST", "/bulk", () => jsonResponse(200, QUEUED_JOB)); + const wm = new WellMarked({ apiKey: API_KEY }); + + await wm.bulk(["https://a.example"]); + await wm.bulk(["https://b.example"]); + + expect(mock.calls[0]!.headers["idempotency-key"]).not.toBe( + mock.calls[1]!.headers["idempotency-key"], + ); + }); + + it("crawl() sends an Idempotency-Key", async () => { + mock.on("POST", "/crawl", () => + jsonResponse(200, { ...QUEUED_JOB, kind: "crawl", total: 0 }), + ); + const wm = new WellMarked({ apiKey: API_KEY }); + + await wm.crawl("https://a.example"); + + expect(mock.calls[0]!.headers["idempotency-key"]).toBeTruthy(); + }); + + it("a client-wide header cannot pin Idempotency-Key across submissions", async () => { + // setHeader is client-wide, so it's the wrong channel for idempotency. + // The per-request key must win, or every later bulk() would replay the + // first job. + mock.on("POST", "/bulk", () => jsonResponse(200, QUEUED_JOB)); + const wm = new WellMarked({ apiKey: API_KEY }); + wm.setHeader("Idempotency-Key", "pinned-forever"); + + await wm.bulk(["https://a.example"]); + await wm.bulk(["https://b.example"]); + + expect(mock.calls[0]!.headers["idempotency-key"]).not.toBe("pinned-forever"); + expect(mock.calls[0]!.headers["idempotency-key"]).not.toBe( + mock.calls[1]!.headers["idempotency-key"], + ); + }); + + it("per-request headers still cannot override Authorization", async () => { + mock.on("POST", "/bulk", () => jsonResponse(200, QUEUED_JOB)); + const wm = new WellMarked({ apiKey: API_KEY }); + + await wm.bulk(["https://a.example"], { idempotencyKey: "k1" }); + + expect(mock.calls[0]!.headers["authorization"]).toBe(`Bearer ${API_KEY}`); + }); +}); + +// ── Internal retry ───────────────────────────────────────────────────────── +// Retries exist so the auto-generated Idempotency-Key is worth something: a +// connection blip is ambiguous (the job may already exist), and replaying with +// the SAME key collapses the attempts into one job instead of two. + +describe("retry", () => { + it("retries a connection failure on bulk and reuses the same key", async () => { + let attempts = 0; + const seenKeys: string[] = []; + const failingFetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + attempts++; + const h = init!.headers as Record; + seenKeys.push(h["Idempotency-Key"]!); + if (attempts === 1) throw new TypeError("network down"); + return jsonResponse(200, QUEUED_JOB); + }) as typeof fetch; + + vi.stubGlobal("fetch", failingFetch); + const wm = new WellMarked({ apiKey: API_KEY }); + const job = await wm.bulk(["https://a.example"]); + + expect(attempts).toBe(2); + expect(job.status).toBe("queued"); + // The whole point: both attempts carry the SAME key, so the API replays + // rather than creating a second job. + expect(seenKeys[0]).toBe(seenKeys[1]); + }); + + it("does NOT retry POST /extract — the API bills it on arrival", async () => { + // A connection error can't tell us whether the extraction happened. + // /extract takes no Idempotency-Key, so replaying could bill twice. + let attempts = 0; + const failingFetch = (async () => { + attempts++; + throw new TypeError("network down"); + }) as typeof fetch; + + vi.stubGlobal("fetch", failingFetch); + const wm = new WellMarked({ apiKey: API_KEY }); + await expect(wm.extract("https://a.example")).rejects.toBeInstanceOf(APIConnectionError); + expect(attempts).toBe(1); + }); + + it("retries 5xx but not 4xx", async () => { + let attempts = 0; + const flaky = (async () => { + attempts++; + if (attempts === 1) return jsonResponse(503, { error: { code: "x", message: "down" } }); + return jsonResponse(200, QUEUED_JOB); + }) as typeof fetch; + vi.stubGlobal("fetch", flaky); + const wm = new WellMarked({ apiKey: API_KEY }); + await wm.bulk(["https://a.example"]); + expect(attempts).toBe(2); + + let attempts4xx = 0; + const deterministic = (async () => { + attempts4xx++; + return jsonResponse(422, { + error: { code: "bulk_cap_exceeded", message: "too many" }, + }); + }) as typeof fetch; + vi.stubGlobal("fetch", deterministic); + const wm2 = new WellMarked({ apiKey: API_KEY }); + await expect(wm2.bulk(["https://a.example"])).rejects.toBeInstanceOf( + UnprocessableEntityError, + ); + // 4xx is deterministic — replaying just reproduces it. + expect(attempts4xx).toBe(1); + }); + + it("gives up after maxRetries and surfaces the connection error", async () => { + let attempts = 0; + const alwaysDown = (async () => { + attempts++; + throw new TypeError("network down"); + }) as typeof fetch; + + vi.stubGlobal("fetch", alwaysDown); + const wm = new WellMarked({ apiKey: API_KEY, maxRetries: 1 }); + await expect(wm.bulk(["https://a.example"])).rejects.toBeInstanceOf(APIConnectionError); + expect(attempts).toBe(2); + }); + + it("maxRetries: 0 disables retrying entirely", async () => { + let attempts = 0; + const alwaysDown = (async () => { + attempts++; + throw new TypeError("network down"); + }) as typeof fetch; + + vi.stubGlobal("fetch", alwaysDown); + const wm = new WellMarked({ apiKey: API_KEY, maxRetries: 0 }); + await expect(wm.bulk(["https://a.example"])).rejects.toBeInstanceOf(APIConnectionError); + expect(attempts).toBe(1); + }); + + it("a client-wide Idempotency-Key must NOT make /extract retryable", async () => { + // Regression: safeToReplay once read the MERGED headers. Idempotency-Key + // isn't reserved, so setHeader() could smuggle one onto every request — + // making POST /extract look replay-safe. The API bills /extract on + // arrival and ignores the header, so retrying it double-charges. + let attempts = 0; + const failingFetch = (async () => { + attempts++; + throw new TypeError("network down"); + }) as typeof fetch; + + vi.stubGlobal("fetch", failingFetch); + const wm = new WellMarked({ apiKey: API_KEY }); + wm.setHeader("Idempotency-Key", "smuggled"); + + await expect(wm.extract("https://a.example")).rejects.toBeInstanceOf(APIConnectionError); + expect(attempts).toBe(1); + }); + + it("a constructor-level Idempotency-Key must NOT make /extract retryable", async () => { + let attempts = 0; + const failingFetch = (async () => { + attempts++; + throw new TypeError("network down"); + }) as typeof fetch; + + vi.stubGlobal("fetch", failingFetch); + const wm = new WellMarked({ + apiKey: API_KEY, + headers: { "Idempotency-Key": "smuggled" }, + }); + + await expect(wm.extract("https://a.example")).rejects.toBeInstanceOf(APIConnectionError); + expect(attempts).toBe(1); + }); +}); + +// ── Phase 5 continuity: policy overrides, key CRUD, logs ───────────────────── + +describe("policy overrides", () => { + it("sends allow_domains / deny_patterns / respect_robots on extract", async () => { + mock.on("POST", "/extract", () => + jsonResponse(200, { + markdown: "# ok", + metadata: { url: "https://a.example" }, + request_id: "r1", + }), + ); + const wm = new WellMarked({ apiKey: API_KEY }); + await wm.extract("https://a.example", { + allowDomains: ["a.example"], + denyPatterns: ["*/admin/*"], + respectRobots: "strict", + }); + const sent = mock.calls.at(-1)!.body as Record; + expect(sent.allow_domains).toEqual(["a.example"]); + expect(sent.deny_patterns).toEqual(["*/admin/*"]); + expect(sent.respect_robots).toBe("strict"); + }); + + it("omits policy fields left unset (never clobbers the key's policy)", async () => { + mock.on("POST", "/extract", () => + jsonResponse(200, { + markdown: "# ok", + metadata: { url: "https://a.example" }, + request_id: "r1", + }), + ); + const wm = new WellMarked({ apiKey: API_KEY }); + await wm.extract("https://a.example"); + const sent = mock.calls.at(-1)!.body as Record; + expect("allow_domains" in sent).toBe(false); + expect("deny_patterns" in sent).toBe(false); + expect("respect_robots" in sent).toBe(false); + }); + + it("surfaces a policy denial on extract as PermissionDeniedError", async () => { + mock.on("POST", "/extract", () => + jsonResponse(403, { + error: { code: "domain_denied", message: "denied", retry: false }, + }), + ); + const wm = new WellMarked({ apiKey: API_KEY }); + await expect(wm.extract("https://blocked.example")).rejects.toMatchObject({ + code: "domain_denied", + }); + await expect(wm.extract("https://blocked.example")).rejects.toBeInstanceOf( + PermissionDeniedError, + ); + }); +}); + +describe("key management", () => { + it("createKey posts scopes + name and returns the raw key once", async () => { + mock.on("POST", "/keys", () => + jsonResponse(200, { + id: "k1", + api_key: "wm_" + "b".repeat(40), + name: "ci", + scopes: ["extract"], + created_at: "2026-07-17T00:00:00Z", + }), + ); + const wm = new WellMarked({ apiKey: API_KEY }); + const key = await wm.createKey(["extract"], { name: "ci" }); + expect(key.apiKey.startsWith("wm_")).toBe(true); + expect(key.scopes).toEqual(["extract"]); + expect(mock.calls.at(-1)!.body).toEqual({ scopes: ["extract"], name: "ci" }); + }); + + it("listKeys returns metadata with an active flag", async () => { + mock.on("GET", "/keys", () => + jsonResponse(200, { + keys: [ + { id: "k1", name: "default", scopes: ["*"], created_at: "2026-07-01T00:00:00Z", revoked_at: null }, + { id: "k2", name: "ci", scopes: ["extract"], created_at: "2026-07-02T00:00:00Z", revoked_at: "2026-07-03T00:00:00Z" }, + ], + }), + ); + const wm = new WellMarked({ apiKey: API_KEY }); + const keys = await wm.listKeys(); + expect(keys.map((k) => k.id)).toEqual(["k1", "k2"]); + expect(keys[0]!.active).toBe(true); + expect(keys[1]!.active).toBe(false); + }); + + it("revokeKey issues a DELETE and returns the revocation", async () => { + mock.on("DELETE", "/keys/k2", () => + jsonResponse(200, { id: "k2", revoked_at: "2026-07-03T00:00:00Z" }), + ); + const wm = new WellMarked({ apiKey: API_KEY }); + const revoked = await wm.revokeKey("k2"); + expect(revoked.id).toBe("k2"); + expect(mock.calls.at(-1)!.method).toBe("DELETE"); + }); +}); + +describe("getLogs", () => { + it("passes limit/offset and parses policy_decision + key_id", async () => { + mock.on("GET", "/logs", () => + jsonResponse(200, { + logs: [ + { + id: "r1", + timestamp: "2026-07-17T00:00:00Z", + target_url: "https://a.example", + status_code: 403, + duration_ms: 3, + error_code: "domain_denied", + render_js: false, + key_id: "k1", + policy_decision: "domain_denied", + }, + ], + limit: 50, + offset: 0, + has_more: true, + }), + ); + const wm = new WellMarked({ apiKey: API_KEY }); + const page = await wm.getLogs({ limit: 50, offset: 0 }); + expect(page.hasMore).toBe(true); + expect(page.logs[0]!.policyDecision).toBe("domain_denied"); + expect(page.logs[0]!.keyId).toBe("k1"); + expect(mock.calls.at(-1)!.url).toContain("limit=50"); + expect(mock.calls.at(-1)!.url).toContain("offset=0"); + }); +}); + +// ── Phase 6: self-registration ─────────────────────────────────────────────── + +describe("register (static)", () => { + it("posts email with no auth header and returns the account", async () => { + mock.on("POST", "/register", () => + jsonResponse(200, { + api_key: "wm_" + "d".repeat(40), + user_id: "u1", + plan: "free", + scopes: ["extract"], + }), + ); + const account = await WellMarked.register("agent@example.com"); + expect(account.apiKey.startsWith("wm_")).toBe(true); + expect(account.plan).toBe("free"); + expect(account.scopes).toEqual(["extract"]); + const sent = mock.calls.at(-1)!; + expect("authorization" in sent.headers).toBe(false); + expect(sent.body).toEqual({ email: "agent@example.com" }); + }); + + it("throws RateLimitError on register_rate_limited", async () => { + mock.on("POST", "/register", () => + jsonResponse(429, { + error: { code: "register_rate_limited", message: "slow down", retry: true }, + }), + ); + await expect( + WellMarked.register("agent@example.com"), + ).rejects.toMatchObject({ code: "register_rate_limited" }); + }); +}); + +// ── Search ───────────────────────────────────────────────────────────────── + +describe("search", () => { + const SEARCH_BODY = { + query: "typescript generics", + results: [ + { url: "https://a.test/1", status: "ok", title: "A", snippet: "s1", markdown: "# A" }, + { url: "https://b.test/2", status: "error", title: "B", snippet: "s2", error: "target_timeout" }, + ], + request_id: "33333333-3333-3333-3333-333333333333", + }; + + it("returns parsed results and sends the expected payload", async () => { + mock.on("POST", "/search", () => jsonResponse(200, SEARCH_BODY)); + + const wm = new WellMarked({ apiKey: API_KEY }); + const res = await wm.search("typescript generics", { numResults: 2 }); + + // Request shape reached the server unchanged (format defaults to markdown; + // policy overrides are omitted entirely when unset). + expect(mock.calls.at(-1)?.body).toEqual({ + query: "typescript generics", + num_results: 2, + render_js: false, + format: "markdown", + }); + + expect(res.query).toBe("typescript generics"); + expect(res.requestId).toBe("33333333-3333-3333-3333-333333333333"); + expect(res.results).toHaveLength(2); + const ok = res.results[0]!; + const err = res.results[1]!; + expect(ok.ok).toBe(true); + expect(ok.markdown).toBe("# A"); + expect(ok.title).toBe("A"); + // A failed page still carries the provider snippet + a stable error code. + expect(err.ok).toBe(false); + expect(err.error).toBe("target_timeout"); + expect(err.snippet).toBe("s2"); + }); + + it("defaults num_results to 5 when omitted", async () => { + mock.on("POST", "/search", () => jsonResponse(200, { ...SEARCH_BODY, results: [] })); + const wm = new WellMarked({ apiKey: API_KEY }); + await wm.search("q"); + expect((mock.calls.at(-1)?.body as { num_results: number }).num_results).toBe(5); + }); + + it("throws PermissionDeniedError on the Pro+ plan gate", async () => { + mock.on("POST", "/search", () => + jsonResponse(403, { error: { code: "plan_not_supported", message: "Pro+ only." } }), + ); + const wm = new WellMarked({ apiKey: API_KEY }); + await expect(wm.search("q")).rejects.toBeInstanceOf(PermissionDeniedError); + await expect(wm.search("q")).rejects.toMatchObject({ code: "plan_not_supported" }); + }); + + it("carries the full extraction parameter set: format + policy overrides", async () => { + mock.on("POST", "/search", () => + jsonResponse(200, { + query: "q", + results: [{ + url: "https://a.test/", status: "ok", snippet: "s", + chunks: [{ text: "hi", start_token: 0, end_token: 2 }], + }], + request_id: "33333333-3333-3333-3333-333333333333", + }), + ); + const wm = new WellMarked({ apiKey: API_KEY }); + const res = await wm.search("q", { + format: "chunks", + allowDomains: ["a.test"], + respectRobots: "strict", + }); + + const sent = mock.calls.at(-1)?.body as Record; + expect(sent.format).toBe("chunks"); + expect(sent.allow_domains).toEqual(["a.test"]); + expect(sent.respect_robots).toBe("strict"); + expect("deny_patterns" in sent).toBe(false); // unset overrides omitted + + const item = res.results[0]!; + expect(item.ok).toBe(true); + expect(item.markdown).toBeNull(); + expect(item.chunks?.[0]).toEqual({ text: "hi", startToken: 0, endToken: 2 }); + expect(contentOf(item)).toBe(item.chunks); // format-agnostic accessor + }); +}); + +// ── Output formats (Phase 4.5) ────────────────────────────────────────────── +// The format param must reach the wire, and each format's payload must land in +// its own field. Content silently arriving as null would look to the caller +// like a successful-but-empty extraction. + +describe("output formats", () => { + it("sends format and parses json blocks + metrics", async () => { + mock.on("POST", "/extract", () => + jsonResponse(200, { + blocks: [ + { type: "heading", text: "Title", level: 1 }, + { type: "paragraph", text: "Body text.", level: null }, + ], + metrics: { + content_bytes: 8000, + input_tokens: 2000, + output_tokens: 500, + tokens_saved: 1500, + reduction_pct: 75.0, + }, + metadata: { url: "https://example.com" }, + request_id: "id", + }), + ); + const wm = new WellMarked({ apiKey: API_KEY }); + const result = await wm.extract("https://example.com", { format: "json" }); + + expect((mock.calls[0]!.body as Record).format).toBe("json"); + expect(result.markdown).toBeNull(); + expect(result.blocks?.map((b) => b.type)).toEqual(["heading", "paragraph"]); + expect(result.blocks?.[0]!.level).toBe(1); + expect(result.blocks?.[1]!.level).toBeNull(); + expect(result.metrics?.tokensSaved).toBe(1500); + expect(result.metrics?.reductionPct).toBe(75.0); + expect(contentOf(result)).toEqual(result.blocks); + }); + + it("parses chunks with contiguous snake_case offsets", async () => { + mock.on("POST", "/extract", () => + jsonResponse(200, { + chunks: [ + { text: "first ", start_token: 0, end_token: 500 }, + { text: "second", start_token: 500, end_token: 812 }, + ], + metadata: { url: "https://example.com" }, + request_id: "id", + }), + ); + const wm = new WellMarked({ apiKey: API_KEY }); + const result = await wm.extract("https://example.com", { format: "chunks" }); + + expect(result.chunks?.map((c) => c.startToken)).toEqual([0, 500]); + // Contiguity survives the snake_case → camelCase mapping. + expect(result.chunks?.[0]!.endToken).toBe(result.chunks?.[1]!.startToken); + }); + + it("forwards format on bulk and crawl", async () => { + mock.on("POST", "/bulk", () => + jsonResponse(200, { + job_id: "j", + kind: "bulk", + status: "queued", + total: 1, + completed: 0, + results: [], + }), + ); + mock.on("POST", "/crawl", () => + jsonResponse(200, { + job_id: "c", + kind: "crawl", + status: "queued", + total: 0, + completed: 0, + truncated: false, + truncated_reason: null, + results: [], + }), + ); + const wm = new WellMarked({ apiKey: API_KEY }); + await wm.bulk(["https://a.test"], { format: "links" }); + await wm.crawl("https://b.test", { format: "html" }); + + expect((mock.calls[0]!.body as Record).format).toBe("links"); + expect((mock.calls[1]!.body as Record).format).toBe("html"); + }); + + it("marks a non-markdown bulk item as ok", () => { + // Negative control: keying `ok` on `markdown !== null` (as it did before + // formats existed) reports every successful links/html/chunks item failed. + const item = bulkItemFromDict({ + url: "https://a.test", + links: ["https://a.test/x"], + error: null, + }); + expect(item.ok).toBe(true); + expect(item.markdown).toBeNull(); + expect(contentOf(item)).toEqual(["https://a.test/x"]); + + const failed = bulkItemFromDict({ + url: "https://b.test", + error: "target_timeout", }); + expect(failed.ok).toBe(false); + expect(contentOf(failed)).toBeNull(); }); }); diff --git a/tsconfig.build.json b/tsconfig.build.json index 624bc7c..8ff3d7b 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -1,6 +1,10 @@ { "extends": "./tsconfig.json", "compilerOptions": { + // The CJS build passes --moduleResolution node (node10), which TS 6 + // deprecates and TS 7 will drop. Silence it for now; migrating the CJS + // output to node16 resolution is a separate, behaviour-affecting change. + "ignoreDeprecations": "6.0", "rootDir": "src", "declaration": true, "declarationMap": false, diff --git a/tsconfig.json b/tsconfig.json index e752a6d..edaf5b5 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,7 +1,10 @@ { "compilerOptions": { "target": "ES2020", - "lib": ["ES2020", "DOM"], + // ES2022 for the type surface only (emit stays at the ES2020 target): + // the package supports Node >=18, which has ES2022 array methods such as + // Array.prototype.at. Without this, TS 6 rejects `.at()` in the tests. + "lib": ["ES2022", "DOM"], "module": "ESNext", "moduleResolution": "Bundler", "strict": true,