From 0475a2ebf5d97aa87c600ebde87bc62a953b7929 Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Sun, 19 Jul 2026 22:26:01 +0530 Subject: [PATCH 1/7] feat(evals): prompt iteration async job --- app/(main)/evaluations/[id]/page.tsx | 53 +--- .../evaluations/[id]/improve-prompt/route.ts | 40 ++- app/api/webhooks/prompt-improvement/route.ts | 52 ++++ .../prompt-improvement/stream/route.ts | 89 +++++++ app/hooks/usePromptImprovement.ts | 227 ++++++++++++++++++ app/lib/store/promptImprovementStore.ts | 115 +++++++++ app/lib/types/promptImprovement.ts | 43 ++++ 7 files changed, 568 insertions(+), 51 deletions(-) create mode 100644 app/api/webhooks/prompt-improvement/route.ts create mode 100644 app/api/webhooks/prompt-improvement/stream/route.ts create mode 100644 app/hooks/usePromptImprovement.ts create mode 100644 app/lib/store/promptImprovementStore.ts create mode 100644 app/lib/types/promptImprovement.ts diff --git a/app/(main)/evaluations/[id]/page.tsx b/app/(main)/evaluations/[id]/page.tsx index f4f4f27..2223a01 100644 --- a/app/(main)/evaluations/[id]/page.tsx +++ b/app/(main)/evaluations/[id]/page.tsx @@ -8,9 +8,9 @@ import { useState, useEffect, useCallback, useRef } from "react"; import { useRouter, useParams } from "next/navigation"; import { apiFetch } from "@/app/lib/apiClient"; -import { invalidateConfigCache } from "@/app/lib/utils"; import { useAuth } from "@/app/lib/context/AuthContext"; import { useApp } from "@/app/lib/context/AppContext"; +import { usePromptImprovement } from "@/app/hooks/usePromptImprovement"; import type { EvalJob, EvalJobApiResponse, @@ -71,8 +71,12 @@ export default function EvaluationReport() { const [isConfigModalOpen, setIsConfigModalOpen] = useState(false); const [exportFormat, setExportFormat] = useState<"row" | "grouped">("row"); const [isResyncing, setIsResyncing] = useState(false); - const [isImprovingPrompt, setIsImprovingPrompt] = useState(false); const [showNoTracesModal, setShowNoTracesModal] = useState(false); + const { isImprovingPrompt, handleIteratePrompt } = usePromptImprovement({ + jobId, + apiKey, + isAuthenticated, + }); const fetchJobDetails = useCallback(async () => { if (!isAuthenticated || !jobId) return; @@ -182,51 +186,6 @@ export default function EvaluationReport() { } }; - const handleIteratePrompt = async () => { - if (!isAuthenticated || !jobId) return; - setIsImprovingPrompt(true); - try { - const data = await apiFetch<{ - success: boolean; - error?: string; - data?: { - config_id?: string; - id?: string; - version?: number; - } | null; - }>(`/api/evaluations/${jobId}/improve-prompt`, apiKey, { - method: "POST", - body: JSON.stringify({}), - }); - - if (!data.success) { - toast.error(data.error || "Failed to iterate on prompt"); - return; - } - - invalidateConfigCache(); - toast.success("Prompt iteration created"); - - const newConfigId = data.data?.config_id ?? data.data?.id; - const newVersion = data.data?.version; - if (newConfigId) { - const versionParam = - typeof newVersion === "number" ? `&version=${newVersion}` : ""; - router.push( - `/configurations/prompt-editor?config=${encodeURIComponent( - newConfigId, - )}${versionParam}&from=evaluations`, - ); - } - } catch (err: unknown) { - toast.error( - `Failed to iterate on prompt: ${err instanceof Error ? err.message : "Unknown error"}`, - ); - } finally { - setIsImprovingPrompt(false); - } - }; - const handleResync = async () => { if (!isAuthenticated || !jobId) return; diff --git a/app/api/evaluations/[id]/improve-prompt/route.ts b/app/api/evaluations/[id]/improve-prompt/route.ts index 3368240..0c00a78 100644 --- a/app/api/evaluations/[id]/improve-prompt/route.ts +++ b/app/api/evaluations/[id]/improve-prompt/route.ts @@ -1,5 +1,20 @@ import { NextRequest, NextResponse } from "next/server"; import { apiClient } from "@/app/lib/apiClient"; +import { markJobPending } from "@/app/lib/store/promptImprovementStore"; +import type { LLMJobImmediatePublic } from "@/app/lib/types/promptImprovement"; + +function resolveCallbackUrl(request: NextRequest): string { + const configured = + process.env.NEXT_PUBLIC_APP_URL ?? process.env.APP_URL ?? null; + const base = configured + ? configured.replace(/\/$/, "") + : `${request.headers.get("x-forwarded-proto") ?? "https"}://${ + request.headers.get("x-forwarded-host") ?? + request.headers.get("host") ?? + request.nextUrl.host + }`; + return `${base}/api/webhooks/prompt-improvement`; +} export async function POST( request: NextRequest, @@ -8,22 +23,39 @@ export async function POST( try { const { id } = await params; - let body: unknown = undefined; + let userBody: Record = {}; try { - body = await request.json(); + const parsed = await request.json(); + if (parsed && typeof parsed === "object") { + userBody = parsed as Record; + } } catch { - body = {}; + userBody = {}; } + const payload = { + ...userBody, + callback_url: userBody.callback_url ?? resolveCallbackUrl(request), + }; + const { status, data } = await apiClient( request, `/api/v1/evaluations/${id}/improve-prompt`, { method: "POST", - body: JSON.stringify(body ?? {}), + body: JSON.stringify(payload), }, ); + if (status === 202 && data && typeof data === "object") { + const envelope = data as { + success?: boolean; + data?: LLMJobImmediatePublic; + }; + const jobId = envelope.data?.job_id; + if (jobId) markJobPending(jobId); + } + return NextResponse.json(data ?? { error: "Backend error" }, { status }); } catch (error: unknown) { console.error("improve-prompt proxy error:", error); diff --git a/app/api/webhooks/prompt-improvement/route.ts b/app/api/webhooks/prompt-improvement/route.ts new file mode 100644 index 0000000..efe8cd9 --- /dev/null +++ b/app/api/webhooks/prompt-improvement/route.ts @@ -0,0 +1,52 @@ +import { NextRequest, NextResponse } from "next/server"; +import { + getJobSnapshot, + saveJobSnapshot, +} from "@/app/lib/store/promptImprovementStore"; +import type { PromptImprovementJobPublic } from "@/app/lib/types/promptImprovement"; + +export async function POST(request: NextRequest) { + try { + const body = (await request.json()) as { + success?: boolean; + data?: PromptImprovementJobPublic; + error?: string | null; + }; + if (!body.data || !body.data.job_id) { + return NextResponse.json( + { success: false, error: "invalid_payload" }, + { status: 400 }, + ); + } + saveJobSnapshot(body.data); + return NextResponse.json({ success: true, data: { received: true } }); + } catch (error) { + console.error("prompt-improvement webhook error:", error); + return NextResponse.json( + { + success: false, + error: + error instanceof Error ? error.message : "webhook_processing_failed", + }, + { status: 500 }, + ); + } +} + +export async function GET(request: NextRequest) { + const jobId = request.nextUrl.searchParams.get("job_id"); + if (!jobId) { + return NextResponse.json( + { success: false, error: "job_id_required" }, + { status: 400 }, + ); + } + const snapshot = getJobSnapshot(jobId); + if (!snapshot) { + return NextResponse.json( + { success: false, error: "job_not_found", data: null }, + { status: 404 }, + ); + } + return NextResponse.json({ success: true, data: snapshot }); +} diff --git a/app/api/webhooks/prompt-improvement/stream/route.ts b/app/api/webhooks/prompt-improvement/stream/route.ts new file mode 100644 index 0000000..0fc4e68 --- /dev/null +++ b/app/api/webhooks/prompt-improvement/stream/route.ts @@ -0,0 +1,89 @@ +import { NextRequest } from "next/server"; +import { + getJobSnapshot, + subscribeJobSnapshot, +} from "@/app/lib/store/promptImprovementStore"; +import type { PromptImprovementJobSnapshot } from "@/app/lib/types/promptImprovement"; + +export const runtime = "nodejs"; +export const dynamic = "force-dynamic"; + +export async function GET(request: NextRequest) { + const jobId = request.nextUrl.searchParams.get("job_id"); + if (!jobId) { + return new Response("job_id_required", { status: 400 }); + } + + const encoder = new TextEncoder(); + const stream = new ReadableStream({ + start(controller) { + const send = (event: string, data: unknown) => { + try { + controller.enqueue( + encoder.encode( + `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`, + ), + ); + } catch { + // stream closed + } + }; + + const finish = () => { + clearInterval(heartbeat); + unsubscribe(); + try { + controller.close(); + } catch { + // already closed + } + }; + + const push = (snap: PromptImprovementJobSnapshot) => { + send("snapshot", snap); + if (snap.status === "SUCCESS" || snap.status === "FAILED") { + finish(); + } + }; + + // Prime with current state (or a synthetic PENDING when store hasn't + // seen it yet — keeps the client from thinking the stream is stalled). + const current = getJobSnapshot(jobId); + if (current) { + push(current); + } else { + send("snapshot", { + job_id: jobId, + status: "PENDING", + config_version: null, + error_message: null, + updated_at: new Date().toISOString(), + }); + } + + const unsubscribe = subscribeJobSnapshot(jobId, push); + + // every 20s so intermediaries (load balancers, browsers) don't + // consider the connection idle. + const heartbeat = setInterval(() => { + try { + controller.enqueue(encoder.encode(`: ping\n\n`)); + } catch { + finish(); + } + }, 20000); + + const abort = () => finish(); + request.signal.addEventListener("abort", abort); + }, + }); + + return new Response(stream, { + headers: { + "Content-Type": "text/event-stream; charset=utf-8", + "Cache-Control": "no-cache, no-transform", + Connection: "keep-alive", + "X-Accel-Buffering": "no", + }, + }); +} diff --git a/app/hooks/usePromptImprovement.ts b/app/hooks/usePromptImprovement.ts new file mode 100644 index 0000000..fcf5da6 --- /dev/null +++ b/app/hooks/usePromptImprovement.ts @@ -0,0 +1,227 @@ +"use client"; + +import { useCallback, useEffect, useRef, useState } from "react"; +import { useRouter } from "next/navigation"; +import { apiFetch } from "@/app/lib/apiClient"; +import { invalidateConfigCache } from "@/app/lib/utils"; +import { useToast } from "@/app/hooks/useToast"; +import type { + LLMJobImmediatePublic, + PromptImprovementConfigVersion, + UsePromptImprovementArgs, + UsePromptImprovementResult, +} from "@/app/lib/types/promptImprovement"; + +type IterateSettle = ( + result: + | { status: "SUCCESS"; version: PromptImprovementConfigVersion } + | { status: "FAILED"; message: string | null }, +) => void; + +interface JobSnapshotLike { + status: string; + config_version: PromptImprovementConfigVersion | null; + error_message: string | null; +} + +/** Map a sync validation code (from the initial 202 request) to a user-facing message. */ +function iteratePromptSyncError(code: string): string { + switch (code) { + case "evaluation_not_found": + return "Evaluation not found"; + case "evaluation_not_completed": + return "Prompt iteration is only available for completed evaluations"; + case "source_config_unavailable": + return "Source config is no longer available for this evaluation"; + case "traces_not_available": + return "Evaluation traces aren't available yet — try Resync first"; + case "invalid_callback_url": + return "Frontend callback URL is invalid. Set NEXT_PUBLIC_APP_URL to a public HTTPS host."; + case "prompt_improvement_enqueue_failed": + return "Backend couldn't queue the improvement job. Retry in a moment."; + default: + return code || "Failed to queue prompt iteration"; + } +} + +function readSnapshot(snap: JobSnapshotLike, settle: IterateSettle): boolean { + if (snap.status === "SUCCESS") { + if (!snap.config_version) { + settle({ + status: "FAILED", + message: "Backend reported SUCCESS without a config version.", + }); + return true; + } + settle({ status: "SUCCESS", version: snap.config_version }); + return true; + } + if (snap.status === "FAILED") { + settle({ status: "FAILED", message: snap.error_message ?? null }); + return true; + } + return false; +} + +/** + * Owns the async prompt-improvement flow for an evaluation: fires the initial + * POST, keeps a live channel open (SSE first, poll fallback), settles on + * SUCCESS/FAILED, and navigates to the new config version. Cleans up any + * open interval / EventSource on unmount. + */ +export function usePromptImprovement({ + jobId, + apiKey, + isAuthenticated, +}: UsePromptImprovementArgs): UsePromptImprovementResult { + const router = useRouter(); + const toast = useToast(); + const [isImprovingPrompt, setIsImprovingPrompt] = useState(false); + const pollRef = useRef | null>(null); + const sourceRef = useRef(null); + + const stopWatch = useCallback(() => { + if (pollRef.current) { + clearInterval(pollRef.current); + pollRef.current = null; + } + if (sourceRef.current) { + sourceRef.current.close(); + sourceRef.current = null; + } + }, []); + + useEffect(() => () => stopWatch(), [stopWatch]); + + const pollJob = useCallback( + (pollJobId: string, settle: IterateSettle) => { + const tick = async () => { + try { + const res = await fetch( + `/api/webhooks/prompt-improvement?job_id=${encodeURIComponent(pollJobId)}`, + ); + if (res.status === 404 || !res.ok) return; + const body = (await res.json()) as { + success: boolean; + data?: JobSnapshotLike; + }; + if ( + body.data && + readSnapshot(body.data, (result) => { + stopWatch(); + settle(result); + }) + ) { + // terminal — stopWatch already ran + } + } catch (e) { + console.error("iterate poll error:", e); + } + }; + tick(); + pollRef.current = setInterval(tick, 3000); + }, + [stopWatch], + ); + + const watchJob = useCallback( + (watchJobId: string, settle: IterateSettle) => { + if (typeof window === "undefined" || typeof EventSource === "undefined") { + pollJob(watchJobId, settle); + return; + } + const source = new EventSource( + `/api/webhooks/prompt-improvement/stream?job_id=${encodeURIComponent(watchJobId)}`, + ); + sourceRef.current = source; + let fellBack = false; + const doFallback = () => { + if (fellBack) return; + fellBack = true; + source.close(); + sourceRef.current = null; + pollJob(watchJobId, settle); + }; + source.addEventListener("snapshot", (e) => { + try { + const snap = JSON.parse((e as MessageEvent).data) as JobSnapshotLike; + readSnapshot(snap, (result) => { + stopWatch(); + settle(result); + }); + } catch (err) { + console.error("iterate SSE parse error:", err); + } + }); + source.addEventListener("error", () => { + // CLOSED = server hung up. CONNECTING = browser is retrying, leave it. + if (source.readyState === EventSource.CLOSED) doFallback(); + }); + }, + [pollJob, stopWatch], + ); + + const settleIteration: IterateSettle = useCallback( + (result) => { + setIsImprovingPrompt(false); + if (result.status === "SUCCESS") { + invalidateConfigCache(); + toast.success("Prompt iteration ready"); + const cfg = result.version; + if (cfg.config_id) { + router.push( + `/configurations/prompt-editor?config=${encodeURIComponent( + cfg.config_id, + )}&version=${cfg.version}&from=evaluations`, + ); + } + return; + } + toast.error( + result.message + ? `Prompt iteration failed: ${result.message}` + : "Prompt iteration failed", + ); + }, + [router, toast], + ); + + const handleIteratePrompt = useCallback(async () => { + if (!isAuthenticated || !jobId) return; + stopWatch(); + setIsImprovingPrompt(true); + try { + const data = await apiFetch<{ + success: boolean; + error?: string; + data?: LLMJobImmediatePublic | null; + }>(`/api/evaluations/${jobId}/improve-prompt`, apiKey, { + method: "POST", + body: JSON.stringify({}), + }); + + if (!data.success || !data.data?.job_id) { + toast.error(iteratePromptSyncError(data.error || "")); + setIsImprovingPrompt(false); + return; + } + + toast.success("Prompt iteration queued"); + watchJob(data.data.job_id, settleIteration); + } catch (err: unknown) { + const code = err instanceof Error ? err.message : String(err); + toast.error(iteratePromptSyncError(code)); + setIsImprovingPrompt(false); + } + }, [ + isAuthenticated, + jobId, + apiKey, + stopWatch, + toast, + watchJob, + settleIteration, + ]); + + return { isImprovingPrompt, handleIteratePrompt }; +} diff --git a/app/lib/store/promptImprovementStore.ts b/app/lib/store/promptImprovementStore.ts new file mode 100644 index 0000000..c261953 --- /dev/null +++ b/app/lib/store/promptImprovementStore.ts @@ -0,0 +1,115 @@ +/** + * Server-side in-memory store for prompt-improvement job status. The backend + * pushes results to `/api/webhooks/prompt-improvement`; this store keeps the + * latest snapshot per job_id so the browser can poll a Kaapi-frontend BFF + * endpoint instead of receiving webhooks directly. + * + * CAVEAT: in-memory only. Multiple serverless instances won't share state and + * a cold start drops history. For production either back this with Redis / a + * shared store or add SSE broadcast. Guarded by module-level singleton so a + * single Node process retains state between requests. + */ + +import type { + PromptImprovementJobPublic, + PromptImprovementJobSnapshot, +} from "@/app/lib/types/promptImprovement"; + +const globalKey = Symbol.for("kaapi.promptImprovementStore"); + +type SnapshotListener = (snapshot: PromptImprovementJobSnapshot) => void; + +interface Store { + jobs: Map; + listeners: Map>; +} + +type WithStore = { [globalKey]?: Store }; +const g = globalThis as unknown as WithStore; + +if (!g[globalKey]) { + g[globalKey] = { jobs: new Map(), listeners: new Map() }; +} + +const store = g[globalKey]!; + +function emit(snapshot: PromptImprovementJobSnapshot): void { + const set = store.listeners.get(snapshot.job_id); + if (!set) return; + for (const cb of set) { + try { + cb(snapshot); + } catch (e) { + console.error("prompt-improvement listener error:", e); + } + } +} + +export function subscribeJobSnapshot( + jobId: string, + cb: SnapshotListener, +): () => void { + let set = store.listeners.get(jobId); + if (!set) { + set = new Set(); + store.listeners.set(jobId, set); + } + set.add(cb); + return () => { + const s = store.listeners.get(jobId); + if (!s) return; + s.delete(cb); + if (s.size === 0) store.listeners.delete(jobId); + }; +} + +const MAX_JOBS = 500; +const TTL_MS = 30 * 60 * 1000; // drop entries older than 30 min + +function prune(): void { + const now = Date.now(); + for (const [jobId, snap] of store.jobs.entries()) { + if (now - new Date(snap.updated_at).getTime() > TTL_MS) { + store.jobs.delete(jobId); + } + } + if (store.jobs.size > MAX_JOBS) { + const excess = store.jobs.size - MAX_JOBS; + let dropped = 0; + for (const jobId of store.jobs.keys()) { + store.jobs.delete(jobId); + dropped += 1; + if (dropped >= excess) break; + } + } +} + +export function saveJobSnapshot(job: PromptImprovementJobPublic): void { + const snapshot: PromptImprovementJobSnapshot = { + ...job, + updated_at: new Date().toISOString(), + }; + store.jobs.set(job.job_id, snapshot); + prune(); + emit(snapshot); +} + +export function markJobPending(jobId: string): void { + if (store.jobs.has(jobId)) return; + const snapshot: PromptImprovementJobSnapshot = { + job_id: jobId, + status: "PENDING", + config_version: null, + error_message: null, + updated_at: new Date().toISOString(), + }; + store.jobs.set(jobId, snapshot); + prune(); + emit(snapshot); +} + +export function getJobSnapshot( + jobId: string, +): PromptImprovementJobSnapshot | null { + return store.jobs.get(jobId) ?? null; +} diff --git a/app/lib/types/promptImprovement.ts b/app/lib/types/promptImprovement.ts new file mode 100644 index 0000000..f39ced5 --- /dev/null +++ b/app/lib/types/promptImprovement.ts @@ -0,0 +1,43 @@ +export type PromptImprovementStatus = + | "PENDING" + | "PROCESSING" + | "SUCCESS" + | "FAILED"; + +export interface LLMJobImmediatePublic { + job_id: string; + status: PromptImprovementStatus; + message?: string | null; + job_inserted_at?: string; + job_updated_at?: string; +} + +import type { ConfigVersionPublic } from "@/app/lib/types/configs"; + +export type PromptImprovementConfigVersion = ConfigVersionPublic; + +export interface PromptImprovementJobPublic { + job_id: string; + status: PromptImprovementStatus; + config_version: PromptImprovementConfigVersion | null; + error_message: string | null; +} + +export interface PromptImprovementJobSnapshot { + job_id: string; + status: PromptImprovementStatus; + config_version: PromptImprovementConfigVersion | null; + error_message: string | null; + updated_at: string; +} + +export interface UsePromptImprovementArgs { + jobId: string; + apiKey: string; + isAuthenticated: boolean; +} + +export interface UsePromptImprovementResult { + isImprovingPrompt: boolean; + handleIteratePrompt: () => Promise; +} From 5e4c29491ce8d9e16b3660d1d865fdff26de1d6e Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Sun, 19 Jul 2026 23:34:38 +0530 Subject: [PATCH 2/7] fix(*): added the hash key strictly --- .../evaluations/[id]/improve-prompt/route.ts | 9 +++- app/api/webhooks/prompt-improvement/route.ts | 14 ++++++ app/hooks/usePromptImprovement.ts | 2 - app/lib/webhookSecret.ts | 50 +++++++++++++++++++ 4 files changed, 72 insertions(+), 3 deletions(-) create mode 100644 app/lib/webhookSecret.ts diff --git a/app/api/evaluations/[id]/improve-prompt/route.ts b/app/api/evaluations/[id]/improve-prompt/route.ts index 0c00a78..ffbc2f1 100644 --- a/app/api/evaluations/[id]/improve-prompt/route.ts +++ b/app/api/evaluations/[id]/improve-prompt/route.ts @@ -1,9 +1,16 @@ import { NextRequest, NextResponse } from "next/server"; import { apiClient } from "@/app/lib/apiClient"; import { markJobPending } from "@/app/lib/store/promptImprovementStore"; +import { readWebhookSecret } from "@/app/lib/webhookSecret"; import type { LLMJobImmediatePublic } from "@/app/lib/types/promptImprovement"; function resolveCallbackUrl(request: NextRequest): string { + const secretResult = readWebhookSecret(); + if (!secretResult.ok || !secretResult.secret) { + throw new Error( + secretResult.reason ?? "prompt_improvement_webhook_secret_missing", + ); + } const configured = process.env.NEXT_PUBLIC_APP_URL ?? process.env.APP_URL ?? null; const base = configured @@ -13,7 +20,7 @@ function resolveCallbackUrl(request: NextRequest): string { request.headers.get("host") ?? request.nextUrl.host }`; - return `${base}/api/webhooks/prompt-improvement`; + return `${base}/api/webhooks/prompt-improvement?secret_value=${encodeURIComponent(secretResult.secret)}`; } export async function POST( diff --git a/app/api/webhooks/prompt-improvement/route.ts b/app/api/webhooks/prompt-improvement/route.ts index efe8cd9..8fe0ea9 100644 --- a/app/api/webhooks/prompt-improvement/route.ts +++ b/app/api/webhooks/prompt-improvement/route.ts @@ -3,10 +3,24 @@ import { getJobSnapshot, saveJobSnapshot, } from "@/app/lib/store/promptImprovementStore"; +import { readWebhookSecret } from "@/app/lib/webhookSecret"; import type { PromptImprovementJobPublic } from "@/app/lib/types/promptImprovement"; +function isAuthorized(request: NextRequest): boolean { + const secretResult = readWebhookSecret(); + if (!secretResult.ok || !secretResult.secret) return false; + const provided = request.nextUrl.searchParams.get("secret_value"); + return provided === secretResult.secret; +} + export async function POST(request: NextRequest) { try { + if (!isAuthorized(request)) { + return NextResponse.json( + { success: false, error: "unauthorized" }, + { status: 401 }, + ); + } const body = (await request.json()) as { success?: boolean; data?: PromptImprovementJobPublic; diff --git a/app/hooks/usePromptImprovement.ts b/app/hooks/usePromptImprovement.ts index fcf5da6..a762a16 100644 --- a/app/hooks/usePromptImprovement.ts +++ b/app/hooks/usePromptImprovement.ts @@ -205,8 +205,6 @@ export function usePromptImprovement({ setIsImprovingPrompt(false); return; } - - toast.success("Prompt iteration queued"); watchJob(data.data.job_id, settleIteration); } catch (err: unknown) { const code = err instanceof Error ? err.message : String(err); diff --git a/app/lib/webhookSecret.ts b/app/lib/webhookSecret.ts new file mode 100644 index 0000000..48a24a8 --- /dev/null +++ b/app/lib/webhookSecret.ts @@ -0,0 +1,50 @@ +/** + * Shared validation for `PROMPT_IMPROVEMENT_WEBHOOK_SECRET`. Enforces a hex + * secret of at least 32 chars (16 bytes) so misconfigured deploys with + * placeholder / weak values fail closed. + */ + +const HEX_SECRET_REGEX = /^[a-f0-9]{32,}$/i; + +const PLACEHOLDER_VALUES = new Set([ + "changeme", + "change_me", + "secret", + "test", + "password", + "todo", +]); + +export interface WebhookSecretResult { + ok: boolean; + secret?: string; + reason?: + | "prompt_improvement_webhook_secret_missing" + | "prompt_improvement_webhook_secret_placeholder" + | "prompt_improvement_webhook_secret_invalid_format"; +} + +/** + * Read + validate the secret. Returns a discriminated result so callers can + * surface a specific error code instead of a generic 500. + */ +export function readWebhookSecret(): WebhookSecretResult { + const raw = process.env.PROMPT_IMPROVEMENT_WEBHOOK_SECRET; + if (!raw) { + return { ok: false, reason: "prompt_improvement_webhook_secret_missing" }; + } + const trimmed = raw.trim(); + if (PLACEHOLDER_VALUES.has(trimmed.toLowerCase())) { + return { + ok: false, + reason: "prompt_improvement_webhook_secret_placeholder", + }; + } + if (!HEX_SECRET_REGEX.test(trimmed)) { + return { + ok: false, + reason: "prompt_improvement_webhook_secret_invalid_format", + }; + } + return { ok: true, secret: trimmed }; +} From b26cb773153b63d8494531e46439105ed633765a Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Sun, 19 Jul 2026 23:42:23 +0530 Subject: [PATCH 3/7] fix(evals): move the types inside the types folder --- app/hooks/usePromptImprovement.ts | 15 ++------------- app/lib/types/promptImprovement.ts | 12 ++++++++++++ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/app/hooks/usePromptImprovement.ts b/app/hooks/usePromptImprovement.ts index a762a16..b59964a 100644 --- a/app/hooks/usePromptImprovement.ts +++ b/app/hooks/usePromptImprovement.ts @@ -6,24 +6,13 @@ import { apiFetch } from "@/app/lib/apiClient"; import { invalidateConfigCache } from "@/app/lib/utils"; import { useToast } from "@/app/hooks/useToast"; import type { + IterateSettle, + JobSnapshotLike, LLMJobImmediatePublic, - PromptImprovementConfigVersion, UsePromptImprovementArgs, UsePromptImprovementResult, } from "@/app/lib/types/promptImprovement"; -type IterateSettle = ( - result: - | { status: "SUCCESS"; version: PromptImprovementConfigVersion } - | { status: "FAILED"; message: string | null }, -) => void; - -interface JobSnapshotLike { - status: string; - config_version: PromptImprovementConfigVersion | null; - error_message: string | null; -} - /** Map a sync validation code (from the initial 202 request) to a user-facing message. */ function iteratePromptSyncError(code: string): string { switch (code) { diff --git a/app/lib/types/promptImprovement.ts b/app/lib/types/promptImprovement.ts index f39ced5..9701f81 100644 --- a/app/lib/types/promptImprovement.ts +++ b/app/lib/types/promptImprovement.ts @@ -41,3 +41,15 @@ export interface UsePromptImprovementResult { isImprovingPrompt: boolean; handleIteratePrompt: () => Promise; } + +export type IterateSettle = ( + result: + | { status: "SUCCESS"; version: PromptImprovementConfigVersion } + | { status: "FAILED"; message: string | null }, +) => void; + +export interface JobSnapshotLike { + status: string; + config_version: PromptImprovementConfigVersion | null; + error_message: string | null; +} From cc8e5630b65ecd01c5f68f58899a96ea28ac8572 Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Mon, 20 Jul 2026 17:54:27 +0530 Subject: [PATCH 4/7] fix(review): coderabbit suggestion --- app/hooks/usePromptImprovement.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/hooks/usePromptImprovement.ts b/app/hooks/usePromptImprovement.ts index b59964a..1c761aa 100644 --- a/app/hooks/usePromptImprovement.ts +++ b/app/hooks/usePromptImprovement.ts @@ -108,7 +108,7 @@ export function usePromptImprovement({ } }; tick(); - pollRef.current = setInterval(tick, 3000); + pollRef.current = setInterval(tick, 2000); }, [stopWatch], ); From 265400c0f2cbeeaf400eb88b3a33e009e411668a Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Mon, 20 Jul 2026 21:31:42 +0530 Subject: [PATCH 5/7] fix(review): coderabbit suggestion --- app/hooks/usePromptImprovement.ts | 3 +-- app/lib/store/promptImprovementStore.ts | 16 +++------------- app/lib/types/promptImprovement.ts | 7 +++++++ app/lib/webhookSecret.ts | 6 ------ 4 files changed, 11 insertions(+), 21 deletions(-) diff --git a/app/hooks/usePromptImprovement.ts b/app/hooks/usePromptImprovement.ts index 1c761aa..6b77bb4 100644 --- a/app/hooks/usePromptImprovement.ts +++ b/app/hooks/usePromptImprovement.ts @@ -55,8 +55,7 @@ function readSnapshot(snap: JobSnapshotLike, settle: IterateSettle): boolean { /** * Owns the async prompt-improvement flow for an evaluation: fires the initial * POST, keeps a live channel open (SSE first, poll fallback), settles on - * SUCCESS/FAILED, and navigates to the new config version. Cleans up any - * open interval / EventSource on unmount. + * SUCCESS/FAILED, and navigates to the new config version. */ export function usePromptImprovement({ jobId, diff --git a/app/lib/store/promptImprovementStore.ts b/app/lib/store/promptImprovementStore.ts index c261953..9909fbd 100644 --- a/app/lib/store/promptImprovementStore.ts +++ b/app/lib/store/promptImprovementStore.ts @@ -3,28 +3,18 @@ * pushes results to `/api/webhooks/prompt-improvement`; this store keeps the * latest snapshot per job_id so the browser can poll a Kaapi-frontend BFF * endpoint instead of receiving webhooks directly. - * - * CAVEAT: in-memory only. Multiple serverless instances won't share state and - * a cold start drops history. For production either back this with Redis / a - * shared store or add SSE broadcast. Guarded by module-level singleton so a - * single Node process retains state between requests. */ import type { PromptImprovementJobPublic, PromptImprovementJobSnapshot, + PromptImprovementStoreShape, + SnapshotListener, } from "@/app/lib/types/promptImprovement"; const globalKey = Symbol.for("kaapi.promptImprovementStore"); -type SnapshotListener = (snapshot: PromptImprovementJobSnapshot) => void; - -interface Store { - jobs: Map; - listeners: Map>; -} - -type WithStore = { [globalKey]?: Store }; +type WithStore = { [globalKey]?: PromptImprovementStoreShape }; const g = globalThis as unknown as WithStore; if (!g[globalKey]) { diff --git a/app/lib/types/promptImprovement.ts b/app/lib/types/promptImprovement.ts index 9701f81..2b9361c 100644 --- a/app/lib/types/promptImprovement.ts +++ b/app/lib/types/promptImprovement.ts @@ -53,3 +53,10 @@ export interface JobSnapshotLike { config_version: PromptImprovementConfigVersion | null; error_message: string | null; } + +export type SnapshotListener = (snapshot: PromptImprovementJobSnapshot) => void; + +export interface PromptImprovementStoreShape { + jobs: Map; + listeners: Map>; +} diff --git a/app/lib/webhookSecret.ts b/app/lib/webhookSecret.ts index 48a24a8..1459a92 100644 --- a/app/lib/webhookSecret.ts +++ b/app/lib/webhookSecret.ts @@ -1,9 +1,3 @@ -/** - * Shared validation for `PROMPT_IMPROVEMENT_WEBHOOK_SECRET`. Enforces a hex - * secret of at least 32 chars (16 bytes) so misconfigured deploys with - * placeholder / weak values fail closed. - */ - const HEX_SECRET_REGEX = /^[a-f0-9]{32,}$/i; const PLACEHOLDER_VALUES = new Set([ From 4a019161bf93af23872c7ba1ce94e77c6128a74e Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Mon, 20 Jul 2026 21:54:35 +0530 Subject: [PATCH 6/7] fix(evals): added the types --- app/lib/types/webhookSecret.ts | 10 ++++++++++ app/lib/webhookSecret.ts | 14 ++++++-------- 2 files changed, 16 insertions(+), 8 deletions(-) create mode 100644 app/lib/types/webhookSecret.ts diff --git a/app/lib/types/webhookSecret.ts b/app/lib/types/webhookSecret.ts new file mode 100644 index 0000000..957ef57 --- /dev/null +++ b/app/lib/types/webhookSecret.ts @@ -0,0 +1,10 @@ +export type WebhookSecretFailureReason = + | "prompt_improvement_webhook_secret_missing" + | "prompt_improvement_webhook_secret_placeholder" + | "prompt_improvement_webhook_secret_invalid_format"; + +export interface WebhookSecretResult { + ok: boolean; + secret?: string; + reason?: WebhookSecretFailureReason; +} diff --git a/app/lib/webhookSecret.ts b/app/lib/webhookSecret.ts index 1459a92..d75add1 100644 --- a/app/lib/webhookSecret.ts +++ b/app/lib/webhookSecret.ts @@ -9,14 +9,12 @@ const PLACEHOLDER_VALUES = new Set([ "todo", ]); -export interface WebhookSecretResult { - ok: boolean; - secret?: string; - reason?: - | "prompt_improvement_webhook_secret_missing" - | "prompt_improvement_webhook_secret_placeholder" - | "prompt_improvement_webhook_secret_invalid_format"; -} +import type { WebhookSecretResult } from "@/app/lib/types/webhookSecret"; + +export type { + WebhookSecretFailureReason, + WebhookSecretResult, +} from "@/app/lib/types/webhookSecret"; /** * Read + validate the secret. Returns a discriminated result so callers can From 181fdf3d178935562c85d7cf2d95cb1355ca8c2e Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:41:26 +0530 Subject: [PATCH 7/7] fix(evals): few ui updates --- app/(main)/evaluations/[id]/page.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/(main)/evaluations/[id]/page.tsx b/app/(main)/evaluations/[id]/page.tsx index 2223a01..535a88a 100644 --- a/app/(main)/evaluations/[id]/page.tsx +++ b/app/(main)/evaluations/[id]/page.tsx @@ -388,7 +388,7 @@ export default function EvaluationReport() {
-
+
{hasScore && isNewFormat ? ( <>