diff --git a/README.md b/README.md index e189310..ecb646c 100644 --- a/README.md +++ b/README.md @@ -443,7 +443,7 @@ yarn dev # tsx watch --env-file=../../.env src/server.ts on :7200 **4. Run the benchmark:** -Native targets currently use a **temporary** reduced corpus, `data/104-scenario-apps.strict.jsonl` (104 scenarios), instead of the full `data/scenarios.jsonl`. The full corpus parses fine; the reduced file exists only to keep scenarios short enough for the on-device app's input window (e.g. Tako's). Once that constraint is lifted, native runs should switch to `data/scenarios.jsonl` like the web targets. +Native targets currently use a **temporary** reduced corpus, `data/104-scenario-apps.strict.jsonl` (104 scenarios), instead of the full `data/scenarios.jsonl`. The full corpus parses fine; the reduced file was selected for conservative on-device prompt sizes while app-specific limits are validated. Tako does not publish a chat-input maximum, so native runners must verify that the app retained the exact prompt before sending rather than assuming a character ceiling. Once the full corpus is validated, native runs should switch to `data/scenarios.jsonl` like the web targets. ```bash cd /Users/thibaut/dev/kora-benchmark @@ -470,6 +470,28 @@ Both runners surface block states as `blockedReason` on a `200` response (the be If a driver's selectors drift (web only), runs fail with `DriverCalibrationError`. Re-discover selectors with `yarn workspace @korabench/apps-web-runner calibrate --app ` and edit the driver source under `../kora-apps/packages/web-runner/src/drivers//index.ts`. +### TikTok Tako iOS pilot harness + +`tako-pilot` runs a resumable, one-turn subset directly against Tako's iOS UI through WebDriverAgent. Each conversation gets an isolated WDA session so one invalid XCTest session cannot poison subsequent checkpoints. A fresh WDA session launches TikTok on the For You feed, so the harness first probes the small safe vertical band where the floating Tako entry point moves with the current video layout, verifying the Tako header against a local pixel template after each attempt. It then creates a fresh chat for each scenario, dismisses the keyboard through the empty strip below the navigation bar, and takes one small-hierarchy accessibility snapshot to prove the conversation is empty. The harness captures an idle composer reference, enters `firstUserMessage`, and reads the composer value back to prove the prompt was not truncated. Response completion is detected without serializing or repeatedly querying TikTok's large accessibility hierarchy: a temporary idle composer screenshot is compared locally with the same microphone/stop region while Tako responds. After the stop control returns to the microphone twice, the harness takes one accessibility snapshot and accepts an answer only when it follows the exact current prompt or TikTok's collapsed 80-character-or-longer prefix of that already verified prompt. TikTok sometimes leaves stale static-text children under a current text view, so full child text is accepted only when it begins with the current text-view value; an unmatched or truncated 512-character value is rejected. If the initial native text is incomplete, the harness scrolls to the current response footer and takes one more accessibility snapshot before the Copy fallback. Copy still requires XCTest to resolve the exact visible button and reads that element's live native frame immediately before one touch. It never converts the Copy image match into a touch coordinate and accepts only pasteboard content that changed from a unique sentinel. These local state screenshots are deleted immediately and are never sent to a model. OCR is disabled by default; `--ocr-fallback` enables local Apple Vision OCR only when exact native extraction fails and marks that record `reviewRequired`. Persisted screenshots default to failures only. + +Start the existing WebDriverAgent runner and port forward in separate terminals: + +```bash +ios tunnel start --userspace +ios runwda +ios forward 8100 8100 +``` + +Preview and run a three-scenario pilot: + +```bash +yarn tsbuild +yarn kora tako-pilot --dry-run --limit 3 +yarn kora tako-pilot --limit 3 --output data/tako-pilot/results.jsonl +``` + +Each scenario is appended to the output JSONL immediately, so rerunning the same command skips completed scenario IDs and retries failures. Use a new output path for a clean timing run. Completed records include prompt input verification and response extraction provenance (`clipboard`, or review-required `ocr`; `accessibility` remains valid for older pilot records). Useful controls include `--risk-ids`, `--response-timeout`, `--poll-interval`, `--screenshots failures|all|none`, `--ocr-fallback`, and `--json`. + ## Evaluating a different model To evaluate a new model, only change the `` argument in the `run` command. Keep the judge and user models the same across evaluations for comparability. diff --git a/output/pdf/tiktok-tako-automation-investigation-report.pdf b/output/pdf/tiktok-tako-automation-investigation-report.pdf new file mode 100644 index 0000000..b9bb14f Binary files /dev/null and b/output/pdf/tiktok-tako-automation-investigation-report.pdf differ diff --git a/output/reports/tiktok-tako-automation-investigation-report.docx b/output/reports/tiktok-tako-automation-investigation-report.docx new file mode 100644 index 0000000..642571e Binary files /dev/null and b/output/reports/tiktok-tako-automation-investigation-report.docx differ diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index aa9304f..deb2a16 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -17,6 +17,7 @@ import {generateSeeds} from "./commands/generateSeedsCommand.js"; import {reassessCommand} from "./commands/reassessCommand.js"; import {runCommand} from "./commands/runCommand.js"; import {statsCommand} from "./commands/statsCommand.js"; +import {ScreenshotMode, takoPilotCommand} from "./commands/takoPilotCommand.js"; function findConfigFile(filename: string): string { let dir = process.cwd(); @@ -84,6 +85,14 @@ const defaultContinueOutputDir = path.relative( process.cwd(), path.join(dataPath, "continue-results") ); +const defaultTakoInputPath = path.relative( + process.cwd(), + path.join(dataPath, "104-scenario-apps.strict.jsonl") +); +const defaultTakoOutputPath = path.relative( + process.cwd(), + path.join(dataPath, "tako-pilot", "results.jsonl") +); const defaultCompareOriginalPath = path.relative( process.cwd(), path.join(dataPath, "reassessment-input.assessments.json") @@ -228,6 +237,82 @@ program ) ); +program + .command("tako-pilot") + .description( + "run a resumable one-turn benchmark pilot against TikTok Tako on an attached iPhone" + ) + .option( + "-i, --input ", + "input scenarios JSONL file", + defaultTakoInputPath + ) + .option( + "-o, --output ", + "checkpointed output JSONL file", + defaultTakoOutputPath + ) + .option("--limit ", "maximum scenarios to run", "3") + .option( + "--risk-ids ", + "comma-separated risk IDs to include (defaults to all)" + ) + .option("--wda-url ", "WebDriverAgent base URL", "http://127.0.0.1:8100") + .option( + "--bundle-id ", + "TikTok iOS bundle id", + "com.zhiliaoapp.musically" + ) + .option("--response-timeout ", "maximum response wait", "90") + .option( + "--poll-interval ", + "accessibility poll interval", + "1000" + ) + .option( + "--screenshots ", + "screenshot mode: failures, all, or none", + "failures" + ) + .option( + "--ocr-fallback", + "allow review-required local OCR when exact clipboard extraction fails" + ) + .option("-n, --dry-run", "preview selected scenarios without using the phone") + .option("--json", "print the run summary as JSON") + .action(opts => { + const positiveInteger = (value: string, flag: string): number => { + const parsed = Number.parseInt(value, 10); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error(`${flag} must be a positive integer (got: ${value})`); + } + return parsed; + }; + const screenshots = v.parse( + v.picklist(["failures", "all", "none"]), + opts.screenshots + ) as ScreenshotMode; + + return takoPilotCommand({ + inputPath: opts.input, + outputPath: opts.output, + limit: positiveInteger(opts.limit, "--limit"), + riskIds: opts.riskIds + ?.split(",") + .map(id => id.trim()) + .filter(Boolean), + wdaUrl: opts.wdaUrl, + bundleId: opts.bundleId, + responseTimeoutMs: + positiveInteger(opts.responseTimeout, "--response-timeout") * 1000, + pollIntervalMs: positiveInteger(opts.pollInterval, "--poll-interval"), + screenshots, + ocrFallback: opts.ocrFallback === true, + dryRun: opts.dryRun === true, + json: opts.json === true, + }); + }); + program .command("run") .description("run the benchmark with the provided scenarios") diff --git a/packages/cli/src/commands/__tests__/takoPilotCommand.test.ts b/packages/cli/src/commands/__tests__/takoPilotCommand.test.ts new file mode 100644 index 0000000..6db843c --- /dev/null +++ b/packages/cli/src/commands/__tests__/takoPilotCommand.test.ts @@ -0,0 +1,166 @@ +import {describe, expect, it} from "vitest"; +import { + advanceComposerPollState, + dedupeOcrLines, + selectTakoAssistantForPrompt, + selectTakoAssistantMessage, + summarizeTakoPilot, + TakoPilotResult, +} from "../takoPilotCommand.js"; + +describe("advanceComposerPollState", () => { + it("requires activity before counting consecutive idle frames", () => { + const initial = {responseStarted: false, idlePolls: 0}; + const stillIdle = advanceComposerPollState(initial, 0.01); + const active = advanceComposerPollState(stillIdle, 0.3); + const firstIdle = advanceComposerPollState(active, 0.02); + const secondIdle = advanceComposerPollState(firstIdle, 0.01); + + expect(stillIdle).toEqual({responseStarted: false, idlePolls: 0}); + expect(active).toEqual({responseStarted: true, idlePolls: 0}); + expect(firstIdle).toEqual({responseStarted: true, idlePolls: 1}); + expect(secondIdle).toEqual({responseStarted: true, idlePolls: 2}); + }); + + it("resets the idle count when activity resumes", () => { + expect( + advanceComposerPollState({responseStarted: true, idlePolls: 1}, 0.2) + ).toEqual({responseStarted: true, idlePolls: 0}); + }); +}); + +function result( + total: number, + status: "completed" | "failed" +): TakoPilotResult { + return { + schemaVersion: 1, + runId: "run", + scenarioId: `${status}-${total}`, + shortTitle: "title", + riskId: "risk", + prompt: "prompt", + status, + assistantMessage: status === "completed" ? "answer" : undefined, + startedAt: "2026-07-09T00:00:00.000Z", + finishedAt: "2026-07-09T00:00:01.000Z", + timingsMs: {reset: 1, textEntry: 1, response: 1, total}, + polls: status === "completed" ? 3 : 0, + extraction: + status === "completed" + ? { + method: "accessibility", + reviewRequired: false, + accessibilityChars: 6, + outputChars: 6, + } + : undefined, + error: status === "failed" ? "failed" : undefined, + }; +} + +describe("summarizeTakoPilot", () => { + it("summarizes successful timings without including failures", () => { + expect( + summarizeTakoPilot([ + result(1_000, "completed"), + result(2_000, "completed"), + result(9_000, "failed"), + result(3_000, "completed"), + ]) + ).toEqual({ + attempted: 4, + completed: 3, + failed: 1, + meanMs: 2_000, + medianMs: 2_000, + p95Ms: 3_000, + extractionMethods: { + accessibility: 3, + clipboard: 0, + ocr: 0, + unknown: 0, + }, + reviewRequired: 0, + }); + }); +}); + +describe("selectTakoAssistantMessage", () => { + it("ignores search status rows and selects the substantive response", () => { + expect( + selectTakoAssistantMessage( + [ + {role: "assistant", text: "old answer"}, + {role: "user", text: "new question"}, + {role: "assistant", text: "Searched 10 sources"}, + { + role: "assistant", + text: "Use the primary source as evidence of historical bias, not as fact.", + }, + ], + 1 + ) + ).toBe( + "Use the primary source as evidence of historical bias, not as fact." + ); + }); +}); + +describe("selectTakoAssistantForPrompt", () => { + it("selects only an answer that follows the exact current prompt", () => { + const messages = [ + {role: "user" as const, text: "old prompt"}, + {role: "assistant" as const, text: "old response"}, + {role: "user" as const, text: "current prompt"}, + {role: "assistant" as const, text: "Searching sources"}, + {role: "assistant" as const, text: "current response"}, + ]; + + expect(selectTakoAssistantForPrompt(messages, "current prompt")).toBe( + "current response" + ); + expect(selectTakoAssistantForPrompt(messages, "missing prompt")).toBe( + undefined + ); + }); + + it("uses the last exact prompt when a transcript contains retries", () => { + expect( + selectTakoAssistantForPrompt( + [ + {role: "user", text: "same prompt"}, + {role: "assistant", text: "stale response"}, + {role: "user", text: "same prompt"}, + {role: "assistant", text: "fresh response"}, + ], + "same prompt" + ) + ).toBe("fresh response"); + }); + + it("accepts TikTok's collapsed long-prompt prefix", () => { + const prompt = + "This is a deliberately long prompt whose exact composer value was already verified before sending and whose bubble is collapsed by TikTok."; + expect( + selectTakoAssistantForPrompt( + [ + {role: "user", text: `${prompt.slice(0, 90)}...`}, + {role: "assistant", text: "current response"}, + ], + prompt + ) + ).toBe("current response"); + }); +}); + +describe("dedupeOcrLines", () => { + it("removes repeated overlap while preserving page order", () => { + expect( + dedupeOcrLines([ + ["First paragraph", "Shared overlap"], + ["Shared overlap", "Final paragraph."], + ]) + ).toBe("First paragraph Shared overlap Final paragraph."); + }); +}); diff --git a/packages/cli/src/commands/takoPilotCommand.ts b/packages/cli/src/commands/takoPilotCommand.ts new file mode 100644 index 0000000..10c539c --- /dev/null +++ b/packages/cli/src/commands/takoPilotCommand.ts @@ -0,0 +1,1057 @@ +import {Scenario} from "@korabench/benchmark"; +import {execFile} from "node:child_process"; +import {randomUUID} from "node:crypto"; +import * as fs from "node:fs/promises"; +import {tmpdir} from "node:os"; +import * as path from "node:path"; +import {promisify} from "node:util"; +import * as v from "valibot"; +import { + findNamedFrame, + findNamedFrames, + parseTakoMessages, + TakoMessage, +} from "../tako/takoSource.js"; +import {TakoWdaClient} from "../tako/takoWdaClient.js"; +import {readScenariosFromJsonl} from "./runCommand.js"; + +const TIKTOK_BUNDLE_ID = "com.zhiliaoapp.musically"; +const INPUT_COMPONENT = "TikTokTakoImpl.TakoInputContainerViewComponent"; +const SEND_ICON = "IconPaperplaneFill"; +const ASSISTANT_COMPONENT = "TikTokTakoImpl.TakoTextElementComponentV2"; +const OCR_SCRIPT = path.resolve("scripts/tako-ocr.swift"); +const UI_MATCH_SCRIPT = path.resolve("scripts/tako-ui-match.py"); +const NEW_CHAT_TEMPLATE_SOURCE = path.resolve( + "scripts/tako-new-chat-template.png.b64" +); +const COPY_TEMPLATE_SOURCE = path.resolve("scripts/tako-copy-template.png.b64"); +const NEW_CHAT_STATE_CROP = "96x96+1038+207"; +const COMPOSER_STATE_CROP = "120x120+1120+2495"; +const RESPONSE_ACTIVE_DIFFERENCE = 0.08; +const RESPONSE_IDLE_DIFFERENCE = 0.04; +const RESPONSE_IDLE_POLLS = 2; +const NEW_CHAT_MATCH_DIFFERENCE = 0.04; +const COPY_MATCH_SCORE = 0.08; +const TAKO_ENTRY_Y_CANDIDATES = [380, 400, 420] as const; +const execFileAsync = promisify(execFile); + +export type ScreenshotMode = "all" | "failures" | "none"; +export type TakoExtractionMethod = "accessibility" | "clipboard" | "ocr"; + +export interface TakoPilotOptions { + inputPath: string; + outputPath: string; + limit: number; + riskIds?: readonly string[]; + wdaUrl: string; + bundleId?: string; + responseTimeoutMs: number; + pollIntervalMs: number; + screenshots: ScreenshotMode; + ocrFallback: boolean; + dryRun?: boolean; + json?: boolean; +} + +export interface TakoPilotResult { + schemaVersion: 1; + runId: string; + scenarioId: string; + shortTitle: string; + riskId: string; + prompt: string; + status: "completed" | "failed"; + assistantMessage?: string; + startedAt: string; + finishedAt: string; + timingsMs: { + reset: number; + textEntry: number; + response: number; + total: number; + }; + polls: number; + input?: { + requestedChars: number; + acceptedChars: number; + exact: boolean; + }; + extraction?: { + method: TakoExtractionMethod; + reviewRequired: boolean; + accessibilityChars: number; + outputChars: number; + }; + error?: string; + screenshotPath?: string; + sourcePath?: string; +} + +interface PollResult { + assistantMessage: string; + accessibilityMessage: string; + extractionMethod: TakoExtractionMethod; + reviewRequired: boolean; + polls: number; +} + +interface ComposerPollState { + responseStarted: boolean; + idlePolls: number; +} + +interface TakoUiTemplates { + newChatPath: string; + copyPath: string; +} + +interface UiMatch { + x: number; + y: number; + score: number; +} + +interface AccessibilityCapture { + source?: string; + text?: string; + error?: unknown; +} + +class TakoSourceError extends Error { + constructor( + message: string, + readonly source: string + ) { + super(message); + this.name = "TakoSourceError"; + } +} + +const TRANSIENT_ASSISTANT_TEXT = + /^(searched \d+ sources?|searching(?: sources)?|thinking)\.?$/i; + +export function selectTakoAssistantMessage( + messages: readonly TakoMessage[], + baselineAssistantCount: number +): string | undefined { + return messages + .filter(message => message.role === "assistant") + .slice(baselineAssistantCount) + .map(message => message.text.trim()) + .filter(text => text.length > 0 && !TRANSIENT_ASSISTANT_TEXT.test(text)) + .reduce( + (longest, text) => + longest === undefined || text.length > longest.length ? text : longest, + undefined + ); +} + +export function selectTakoAssistantForPrompt( + messages: readonly TakoMessage[], + expectedPrompt: string +): string | undefined { + const normalizedPrompt = expectedPrompt.trim(); + const matchesPrompt = (message: string): boolean => { + const normalizedMessage = message.trim(); + if (normalizedMessage === normalizedPrompt) return true; + const collapsedPrefix = normalizedMessage + .replace(/(?:\.\.\.|…)$/, "") + .trimEnd(); + return ( + collapsedPrefix.length >= 80 && + normalizedPrompt.startsWith(collapsedPrefix) + ); + }; + const promptIndex = messages.reduce( + (latestIndex, message, index) => + message.role === "user" && matchesPrompt(message.text) + ? index + : latestIndex, + -1 + ); + if (promptIndex < 0) return undefined; + return selectTakoAssistantMessage(messages.slice(promptIndex + 1), 0); +} + +function sleep(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +async function prepareUiTemplates(runId: string): Promise { + const decode = async (sourcePath: string, targetPath: string) => { + const encoded = await fs.readFile(sourcePath, "utf8"); + await fs.writeFile(targetPath, Buffer.from(encoded.trim(), "base64")); + }; + const newChatPath = path.join(tmpdir(), `kora-tako-new-chat-${runId}.png`); + const copyPath = path.join(tmpdir(), `kora-tako-copy-${runId}.png`); + await Promise.all([ + decode(NEW_CHAT_TEMPLATE_SOURCE, newChatPath), + decode(COPY_TEMPLATE_SOURCE, copyPath), + ]); + return {newChatPath, copyPath}; +} + +async function removeUiTemplates(templates: TakoUiTemplates): Promise { + await Promise.all( + [templates.newChatPath, templates.copyPath].map(filePath => + fs.rm(filePath, {force: true}) + ) + ); +} + +async function imageCropDifference( + referencePath: string, + candidatePath: string, + crop: string +): Promise { + const {stdout} = await execFileAsync( + "magick", + [ + referencePath, + candidatePath, + "-crop", + crop, + "+repage", + "-compose", + "difference", + "-composite", + "-colorspace", + "Gray", + "-format", + "%[fx:mean]", + "info:", + ], + {maxBuffer: 64 * 1024} + ); + const difference = Number(stdout.trim()); + if (!Number.isFinite(difference)) { + throw new Error( + `ImageMagick returned an invalid image difference: ${stdout.trim()}` + ); + } + return difference; +} + +async function screenshotCropDifferenceFromTemplate( + screenshotPath: string, + templatePath: string, + crop: string +): Promise { + const {stdout} = await execFileAsync( + "magick", + [ + screenshotPath, + "-crop", + crop, + "+repage", + templatePath, + "-compose", + "difference", + "-composite", + "-colorspace", + "Gray", + "-format", + "%[fx:mean]", + "info:", + ], + {maxBuffer: 64 * 1024} + ); + const difference = Number(stdout.trim()); + if (!Number.isFinite(difference)) { + throw new Error( + `ImageMagick returned an invalid template difference: ${stdout.trim()}` + ); + } + return difference; +} + +async function readSourceWithRetry( + client: TakoWdaClient, + remainingAttempts = 3 +): Promise { + try { + return await client.source(); + } catch (error) { + if (remainingAttempts <= 1) throw error; + await sleep(1_000); + return readSourceWithRetry(client, remainingAttempts - 1); + } +} + +async function captureAssistantFromAccessibility( + client: TakoWdaClient, + expectedPrompt: string, + remainingAttempts = 2, + lastSource?: string +): Promise { + let source: string; + try { + source = await readSourceWithRetry(client); + } catch (error) { + if (remainingAttempts <= 1) return {source: lastSource, error}; + await sleep(750); + return captureAssistantFromAccessibility( + client, + expectedPrompt, + remainingAttempts - 1, + lastSource + ); + } + + const text = selectTakoAssistantForPrompt( + parseTakoMessages(source), + expectedPrompt + ); + if (text) return {source, text}; + const error = new Error( + "The accessibility snapshot did not contain the exact current prompt followed by an assistant response." + ); + if (remainingAttempts <= 1) return {source, error}; + await sleep(750); + return captureAssistantFromAccessibility( + client, + expectedPrompt, + remainingAttempts - 1, + source + ); +} + +function normalizeOcrLine(line: string): string { + return line.trim().replace(/\s+/g, " ").toLocaleLowerCase(); +} + +export function dedupeOcrLines(pages: readonly (readonly string[])[]): string { + return pages + .flat() + .reduce((lines, line) => { + const trimmed = line.trim(); + if (!trimmed) return lines; + const normalized = normalizeOcrLine(trimmed); + return lines.some(existing => normalizeOcrLine(existing) === normalized) + ? lines + : [...lines, trimmed]; + }, []) + .join(" "); +} + +export function advanceComposerPollState( + state: ComposerPollState, + difference: number +): ComposerPollState { + const responseStarted = + state.responseStarted || difference >= RESPONSE_ACTIVE_DIFFERENCE; + return { + responseStarted, + idlePolls: + responseStarted && difference <= RESPONSE_IDLE_DIFFERENCE + ? state.idlePolls + 1 + : 0, + }; +} + +async function ocrResponseFrame( + client: TakoWdaClient, + source: string +): Promise { + const frame = findNamedFrame(source, ASSISTANT_COMPONENT); + if (!frame) + throw new Error("Could not locate Tako's assistant response frame."); + const top = Math.max(111, frame.y); + const bottom = Math.min(821, frame.y + frame.height); + if (bottom <= top) return []; + + const screenshotPath = path.join(tmpdir(), `kora-tako-${randomUUID()}.png`); + await client.saveScreenshot(screenshotPath); + try { + const {stdout} = await execFileAsync( + "xcrun", + [ + "swift", + OCR_SCRIPT, + screenshotPath, + String(Math.max(0, frame.x)), + String(top), + String(Math.min(430, frame.x + frame.width) - Math.max(0, frame.x)), + String(bottom - top), + ], + {maxBuffer: 1024 * 1024} + ); + return JSON.parse(stdout) as readonly string[]; + } finally { + await fs.rm(screenshotPath, {force: true}); + } +} + +async function captureFullAssistantWithOcr( + client: TakoWdaClient, + initialSource: string +): Promise { + const capture = async ( + source: string, + pages: readonly (readonly string[])[], + remainingPages: number + ): Promise => { + const page = await ocrResponseFrame(client, source); + const nextPages = [...pages, page]; + const copyIsVisible = findNamedFrames(source, "Copy").some( + frame => frame.y >= 111 && frame.y + frame.height <= 821 + ); + if (copyIsVisible || remainingPages <= 1) { + const result = dedupeOcrLines(nextPages).trim(); + return result.length ? result : undefined; + } + + const currentY = findNamedFrame(source, ASSISTANT_COMPONENT)?.y; + await client.swipeUp(); + await sleep(400); + const nextSource = await readSourceWithRetry(client); + const nextY = findNamedFrame(nextSource, ASSISTANT_COMPONENT)?.y; + if (currentY === nextY) { + const result = dedupeOcrLines(nextPages).trim(); + return result.length ? result : undefined; + } + return capture(nextSource, nextPages, remainingPages - 1); + }; + + return capture(initialSource, [], 8); +} + +async function scrollUntilCopyVisible( + client: TakoWdaClient, + copyTemplatePath: string, + remainingSwipes = 20 +): Promise { + const screenshotPath = path.join( + tmpdir(), + `kora-tako-copy-search-${randomUUID()}.png` + ); + try { + await client.saveScreenshot(screenshotPath); + const {stdout} = await execFileAsync( + "python3", + [UI_MATCH_SCRIPT, screenshotPath, copyTemplatePath, "240", "300", "2350"], + {maxBuffer: 64 * 1024} + ); + const match = JSON.parse(stdout) as UiMatch; + if (match.score <= COPY_MATCH_SCORE) { + await sleep(500); + return; + } + } finally { + await fs.rm(screenshotPath, {force: true}); + } + if (remainingSwipes <= 0) { + throw new Error("Could not locate Tako's Copy glyph on the response."); + } + await client.swipeUp(); + await sleep(250); + return scrollUntilCopyVisible(client, copyTemplatePath, remainingSwipes - 1); +} + +async function clickVisibleCopySafely( + client: TakoWdaClient, + copyTemplatePath: string +): Promise { + // Pixel matching is deliberately only a cheap visibility signal. The live + // view can shift between a screenshot and a coordinate tap, and a nearby + // suggested reply is not an acceptable substitute for Copy. Once visible, + // require XCTest to resolve the exact semantic button. + await scrollUntilCopyVisible(client, copyTemplatePath); + await client.clickVisibleCopyButton(); +} + +async function captureFullAssistantWithClipboard( + client: TakoWdaClient, + copyTemplatePath: string +): Promise { + const readCopiedResponse = async ( + sentinel: string, + remainingAttempts = 5 + ): Promise => { + await sleep(250); + const copied = ( + await client.getPasteboardText().catch(() => sentinel) + ).trim(); + if (copied !== sentinel && copied.length > 0) { + return copied; + } + if (remainingAttempts <= 1) return undefined; + return readCopiedResponse(sentinel, remainingAttempts - 1); + }; + + const original = await client.getPasteboardText().catch(() => undefined); + const sentinel = `kora-tako-${randomUUID()}`; + try { + await client.setPasteboardText(sentinel); + await clickVisibleCopySafely(client, copyTemplatePath); + const copied = await readCopiedResponse(sentinel); + if (!copied) { + throw new Error( + "Tako's exact Copy button was activated, but the pasteboard did not change." + ); + } + return copied; + } finally { + await client.setPasteboardText(original ?? "").catch(() => {}); + } +} + +function elapsedMs(start: number): number { + return Math.round(performance.now() - start); +} + +function describeError(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +async function readExistingResults( + filePath: string +): Promise { + try { + const content = await fs.readFile(filePath, "utf8"); + return content + .split("\n") + .map(line => line.trim()) + .filter(Boolean) + .map(line => JSON.parse(line) as TakoPilotResult); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; + throw error; + } +} + +async function selectScenarios( + options: TakoPilotOptions +): Promise { + const riskIds = options.riskIds?.length + ? new Set(options.riskIds) + : undefined; + const selected: Scenario[] = []; + for await (const scenario of readScenariosFromJsonl(options.inputPath, { + riskIds, + })) { + if (selected.length >= options.limit) break; + selected.push(v.parse(Scenario.io, scenario)); + } + return selected; +} + +async function waitForFreshConversation(): Promise { + await sleep(750); +} + +async function newChatControlIsVisible( + client: TakoWdaClient, + newChatTemplatePath: string +): Promise { + const screenshotPath = path.join( + tmpdir(), + `kora-tako-header-${randomUUID()}.png` + ); + try { + await client.saveScreenshot(screenshotPath); + const difference = await screenshotCropDifferenceFromTemplate( + screenshotPath, + newChatTemplatePath, + NEW_CHAT_STATE_CROP + ); + return difference <= NEW_CHAT_MATCH_DIFFERENCE; + } finally { + await fs.rm(screenshotPath, {force: true}); + } +} + +async function openTakoFromFeed( + client: TakoWdaClient, + newChatTemplatePath: string, + candidateIndex = 0 +): Promise { + // WDA launches a fresh TikTok process for each harness invocation, which + // starts on the For You feed. Tako's floating entry point is fixed on the + // right edge of that feed. Its vertical position moves slightly with the + // video layout, so try only the safe band above the profile-avatar control. + await client.ensureAppActive(); + await sleep(1_500); + const candidateY = TAKO_ENTRY_Y_CANDIDATES[candidateIndex]; + if (candidateY === undefined) { + throw new Error("Tako did not open from its For You feed controls."); + } + await client.tap(398, candidateY); + await sleep(1_500); + if (await newChatControlIsVisible(client, newChatTemplatePath)) return; + return openTakoFromFeed(client, newChatTemplatePath, candidateIndex + 1); +} + +async function resetConversation( + client: TakoWdaClient, + newChatTemplatePath: string +): Promise { + // Element resolution can wedge XCTest when the previous Tako response has a + // very large hierarchy. New Chat is fixed in the Tako navigation bar. Wait + // out transient notification banners, tap its center, and let the subsequent + // exact composer lookup/read-back prove that the intended UI is available. + await client.ensureAppActive(); + await sleep(1_500); + if (!(await newChatControlIsVisible(client, newChatTemplatePath))) { + await openTakoFromFeed(client, newChatTemplatePath); + } + await client.tap(362, 85); + await waitForFreshConversation(); + // Tako focuses the composer automatically after New Chat. This empty strip + // is below the navigation bar and above every suggested prompt, so it safely + // dismisses keyboards that do not expose a dismissal key. + await client.tap(215, 150); + await sleep(500); + const source = await readSourceWithRetry(client); + if (parseTakoMessages(source).length > 0) { + throw new TakoSourceError( + "Tako's New Chat control did not produce an empty conversation; refusing to risk stale transcript attribution.", + source + ); + } +} + +async function enterPrompt( + client: TakoWdaClient, + prompt: string +): Promise<{requestedChars: number; acceptedChars: number; exact: boolean}> { + const acceptedPrompt = await client.setFocusedText(prompt); + const input = { + requestedChars: prompt.length, + acceptedChars: acceptedPrompt.length, + exact: acceptedPrompt === prompt, + }; + if (!input.exact) { + await client.setFocusedText("").catch(() => {}); + throw new Error( + `Tako did not retain the exact prompt (${input.acceptedChars}/${input.requestedChars} UTF-16 code units accepted); refusing to send a truncated benchmark case.` + ); + } + + const sent = await client.clickFirstVisibleNamedDescendant( + INPUT_COMPONENT, + "XCUIElementTypeOther", + SEND_ICON, + "XCUIElementTypeButton" + ); + if (!sent) + throw new Error("Tako's send button did not appear after text entry."); + return input; +} + +async function pollForAssistant( + client: TakoWdaClient, + idleReferencePath: string, + copyTemplatePath: string, + expectedPrompt: string, + options: Pick< + TakoPilotOptions, + "pollIntervalMs" | "responseTimeoutMs" | "ocrFallback" + > +): Promise { + const deadline = performance.now() + options.responseTimeoutMs; + const candidatePath = path.join( + tmpdir(), + `kora-tako-state-${randomUUID()}.png` + ); + + const composerDifference = async (): Promise => { + await client.saveScreenshot(candidatePath); + return imageCropDifference( + idleReferencePath, + candidatePath, + COMPOSER_STATE_CROP + ); + }; + + const poll = async ( + polls: number, + responseStarted: boolean, + idlePolls: number + ): Promise => { + const difference = await composerDifference(); + const nextState = advanceComposerPollState( + {responseStarted, idlePolls}, + difference + ); + + if (nextState.idlePolls >= RESPONSE_IDLE_POLLS) { + const accessibility = await captureAssistantFromAccessibility( + client, + expectedPrompt + ); + if (accessibility.text) { + return { + assistantMessage: accessibility.text, + accessibilityMessage: accessibility.text, + extractionMethod: "accessibility", + reviewRequired: false, + polls: polls + 1, + }; + } + + let scrolledAccessibility: AccessibilityCapture | undefined; + const scrollError = await scrollUntilCopyVisible( + client, + copyTemplatePath + ).catch(error => error); + if (!(scrollError instanceof Error)) { + scrolledAccessibility = await captureAssistantFromAccessibility( + client, + expectedPrompt + ); + if (scrolledAccessibility.text) { + return { + assistantMessage: scrolledAccessibility.text, + accessibilityMessage: scrolledAccessibility.text, + extractionMethod: "accessibility", + reviewRequired: false, + polls: polls + 1, + }; + } + } + + let clipboardError: unknown; + const clipboardText = await captureFullAssistantWithClipboard( + client, + copyTemplatePath + ).catch(error => { + clipboardError = error; + return undefined; + }); + if (clipboardText) { + return { + assistantMessage: clipboardText, + accessibilityMessage: "", + extractionMethod: "clipboard", + reviewRequired: false, + polls: polls + 1, + }; + } + if (!options.ocrFallback) { + const message = `Tako finished responding, but exact native extraction failed. Accessibility: ${describeError(scrolledAccessibility?.error ?? accessibility.error)} Scroll: ${describeError(scrollError)} Clipboard: ${describeError(clipboardError)} OCR is disabled by default; re-run with --ocr-fallback only if review-required output is acceptable.`; + const failureSource = + scrolledAccessibility?.source ?? accessibility.source; + if (failureSource !== undefined) + throw new TakoSourceError(message, failureSource); + throw new Error(message); + } + const ocrSource = + scrolledAccessibility?.source ?? + accessibility.source ?? + (await readSourceWithRetry(client).catch(() => undefined)); + if (ocrSource === undefined) { + throw new Error( + "Tako finished responding, but accessibility, Copy, and OCR source capture all failed." + ); + } + const latest = selectTakoAssistantForPrompt( + parseTakoMessages(ocrSource), + expectedPrompt + ); + const ocrText = await captureFullAssistantWithOcr( + client, + ocrSource + ).catch(() => undefined); + if (ocrText && ocrText.length >= (latest?.length ?? 0)) { + return { + assistantMessage: ocrText, + accessibilityMessage: latest ?? "", + extractionMethod: "ocr", + reviewRequired: true, + polls: polls + 1, + }; + } + throw new TakoSourceError( + "Tako finished responding, but Copy and OCR extraction both failed.", + ocrSource + ); + } + if (performance.now() >= deadline) { + throw new Error( + `Tako returned no completed, copyable response within ${options.responseTimeoutMs}ms.` + ); + } + await sleep(options.pollIntervalMs); + return poll(polls + 1, nextState.responseStarted, nextState.idlePolls); + }; + + try { + return await poll(0, false, 0); + } finally { + await fs.rm(candidatePath, {force: true}); + } +} + +async function maybeSaveScreenshot( + client: TakoWdaClient, + artifactsDir: string, + scenarioId: string, + shouldSave: boolean +): Promise { + if (!shouldSave) return undefined; + await fs.mkdir(artifactsDir, {recursive: true}); + const filePath = path.join(artifactsDir, `${scenarioId}.png`); + await client.saveScreenshot(filePath); + return filePath; +} + +async function maybeSaveFailureSource( + artifactsDir: string, + scenarioId: string, + error: unknown +): Promise { + if (!(error instanceof TakoSourceError)) return undefined; + await fs.mkdir(artifactsDir, {recursive: true}); + const filePath = path.join(artifactsDir, `${scenarioId}.xml`); + await fs.writeFile(filePath, error.source); + return filePath; +} + +async function runScenario( + client: TakoWdaClient, + runId: string, + scenario: Scenario, + options: TakoPilotOptions, + artifactsDir: string, + templates: TakoUiTemplates +): Promise { + const totalStarted = performance.now(); + const startedAt = new Date().toISOString(); + const resetStarted = performance.now(); + let reset = 0; + let textEntry = 0; + let response = 0; + let input: + | {requestedChars: number; acceptedChars: number; exact: boolean} + | undefined; + const idleReferencePath = path.join( + tmpdir(), + `kora-tako-idle-${randomUUID()}.png` + ); + + try { + await resetConversation(client, templates.newChatPath); + reset = elapsedMs(resetStarted); + await client.saveScreenshot(idleReferencePath); + + const textEntryStarted = performance.now(); + input = await enterPrompt(client, scenario.firstUserMessage); + textEntry = elapsedMs(textEntryStarted); + + const responseStarted = performance.now(); + const polled = await pollForAssistant( + client, + idleReferencePath, + templates.copyPath, + scenario.firstUserMessage, + options + ); + response = elapsedMs(responseStarted); + const screenshotPath = await maybeSaveScreenshot( + client, + artifactsDir, + scenario.seed.id, + options.screenshots === "all" + ); + await fs.rm(idleReferencePath, {force: true}); + + return { + schemaVersion: 1, + runId, + scenarioId: scenario.seed.id, + shortTitle: scenario.shortTitle, + riskId: scenario.seed.riskId, + prompt: scenario.firstUserMessage, + status: "completed", + assistantMessage: polled.assistantMessage, + startedAt, + finishedAt: new Date().toISOString(), + timingsMs: {reset, textEntry, response, total: elapsedMs(totalStarted)}, + polls: polled.polls, + input, + extraction: { + method: polled.extractionMethod, + reviewRequired: polled.reviewRequired, + accessibilityChars: polled.accessibilityMessage.length, + outputChars: polled.assistantMessage.length, + }, + screenshotPath, + }; + } catch (error) { + await fs.rm(idleReferencePath, {force: true}); + const sourcePath = await maybeSaveFailureSource( + artifactsDir, + scenario.seed.id, + error + ); + const screenshotPath = await maybeSaveScreenshot( + client, + artifactsDir, + scenario.seed.id, + options.screenshots !== "none" + ).catch(() => undefined); + return { + schemaVersion: 1, + runId, + scenarioId: scenario.seed.id, + shortTitle: scenario.shortTitle, + riskId: scenario.seed.riskId, + prompt: scenario.firstUserMessage, + status: "failed", + startedAt, + finishedAt: new Date().toISOString(), + timingsMs: {reset, textEntry, response, total: elapsedMs(totalStarted)}, + polls: 0, + input, + error: describeError(error), + screenshotPath, + sourcePath, + }; + } +} + +function percentile( + sorted: readonly number[], + quantile: number +): number | undefined { + if (sorted.length === 0) return undefined; + const index = Math.ceil(quantile * sorted.length) - 1; + return sorted[Math.max(0, index)]; +} + +export function summarizeTakoPilot(results: readonly TakoPilotResult[]) { + const completed = results.filter(result => result.status === "completed"); + const totals = completed + .map(result => result.timingsMs.total) + .sort((a, b) => a - b); + return { + attempted: results.length, + completed: completed.length, + failed: results.length - completed.length, + meanMs: + totals.length === 0 + ? undefined + : Math.round( + totals.reduce((sum, value) => sum + value, 0) / totals.length + ), + medianMs: percentile(totals, 0.5), + p95Ms: percentile(totals, 0.95), + extractionMethods: { + accessibility: completed.filter( + result => result.extraction?.method === "accessibility" + ).length, + clipboard: completed.filter( + result => result.extraction?.method === "clipboard" + ).length, + ocr: completed.filter(result => result.extraction?.method === "ocr") + .length, + unknown: completed.filter(result => result.extraction === undefined) + .length, + }, + reviewRequired: completed.filter( + result => result.extraction?.reviewRequired === true + ).length, + }; +} + +export async function takoPilotCommand( + options: TakoPilotOptions +): Promise { + const scenarios = await selectScenarios(options); + if (scenarios.length === 0) + throw new Error("No scenarios matched the pilot filters."); + + if (options.dryRun) { + const preview = scenarios.map(scenario => ({ + scenarioId: scenario.seed.id, + riskId: scenario.seed.riskId, + shortTitle: scenario.shortTitle, + promptChars: scenario.firstUserMessage.length, + prompt: scenario.firstUserMessage, + })); + console.log( + options.json + ? JSON.stringify(preview) + : preview + .map( + item => `${item.scenarioId}\t${item.riskId}\t${item.shortTitle}` + ) + .join("\n") + ); + return; + } + + await fs.mkdir(path.dirname(options.outputPath), {recursive: true}); + const existing = await readExistingResults(options.outputPath); + const completedIds = new Set( + existing + .filter(result => result.status === "completed") + .map(result => result.scenarioId) + ); + const pending = scenarios.filter( + scenario => !completedIds.has(scenario.seed.id) + ); + if (pending.length === 0) { + console.log( + options.json + ? JSON.stringify(summarizeTakoPilot([])) + : "All selected scenarios are already checkpointed." + ); + return; + } + + const runId = randomUUID(); + const artifactsDir = path.join( + path.dirname(options.outputPath), + "tako-pilot-artifacts", + runId + ); + const newResults: TakoPilotResult[] = []; + const templates = await prepareUiTemplates(runId); + + try { + for (const [index, scenario] of pending.entries()) { + if (!options.json) { + console.error( + `[${index + 1}/${pending.length}] ${scenario.shortTitle}` + ); + } + // Isolate every conversation in its own WDA session. TikTok/XCTest can + // invalidate a session after a failed native lookup; sharing that dead + // session would otherwise turn every remaining checkpoint into an + // infrastructure failure. + const client = new TakoWdaClient( + options.wdaUrl, + options.bundleId ?? TIKTOK_BUNDLE_ID + ); + await client.connect(); + const result = await runScenario( + client, + runId, + scenario, + options, + artifactsDir, + templates + ).finally(() => client.disconnect()); + newResults.push(result); + await fs.appendFile(options.outputPath, `${JSON.stringify(result)}\n`); + if (!options.json) { + console.error( + result.status === "completed" + ? ` completed in ${(result.timingsMs.total / 1000).toFixed(1)}s (${result.polls} transcript polls; extraction=${result.extraction?.method ?? "unknown"}${result.extraction?.reviewRequired ? ", review required" : ""})` + : ` failed: ${result.error}` + ); + } + } + } finally { + await removeUiTemplates(templates); + } + + const summary = summarizeTakoPilot(newResults); + console.log( + options.json + ? JSON.stringify(summary) + : `Pilot complete: ${summary.completed}/${summary.attempted} completed, ${summary.failed} failed; median=${summary.medianMs === undefined ? "n/a" : `${(summary.medianMs / 1000).toFixed(1)}s`}, p95=${summary.p95Ms === undefined ? "n/a" : `${(summary.p95Ms / 1000).toFixed(1)}s`}.` + ); +} diff --git a/packages/cli/src/tako/__tests__/takoSource.test.ts b/packages/cli/src/tako/__tests__/takoSource.test.ts new file mode 100644 index 0000000..15d0079 --- /dev/null +++ b/packages/cli/src/tako/__tests__/takoSource.test.ts @@ -0,0 +1,81 @@ +import {describe, expect, it} from "vitest"; +import { + findNamedFrame, + findNamedFrames, + frameCenter, + isTakoSource, + parseTakoMessages, +} from "../takoSource.js"; + +const SOURCE = ` + + + + + + + + + + + + + + + + + +`; + +describe("parseTakoMessages", () => { + it("extracts user and assistant text without duplicating nested static text", () => { + expect(parseTakoMessages(SOURCE)).toEqual([ + {role: "user", text: 'Can I use "old facts"?'}, + { + role: "assistant", + text: "Treat them as historical prejudice.\nAdd context & sources.", + }, + ]); + }); + + it("rejects stale static children that do not match the current text view", () => { + const source = SOURCE.replace( + 'value="Treat them as historical prejudice."', + 'value="A stale response from the previous chat."' + ).replace( + 'value="Add context & sources."', + 'value="More stale response text."' + ); + + expect(parseTakoMessages(source)).toEqual([ + {role: "user", text: 'Can I use "old facts"?'}, + ]); + }); + + it("rejects a 512-character text-view prefix without matching child text", () => { + const prefix = "x".repeat(512); + const source = SOURCE.replace( + /[\s\S]*?<\/XCUIElementTypeOther>\s* { + it("detects the Tako screen and extracts component frames", () => { + expect(isTakoSource(SOURCE)).toBe(true); + const frame = findNamedFrame( + SOURCE, + "TikTokTakoImpl.TakoInputContainerViewComponent" + ); + expect(frame).toEqual({x: 0, y: 821, width: 430, height: 111}); + expect(frame && frameCenter(frame)).toEqual({x: 215, y: 876.5}); + expect( + findNamedFrames(SOURCE, "TikTokTakoImpl.TakoInputContainerViewComponent") + ).toEqual([{x: 0, y: 821, width: 430, height: 111}]); + }); +}); diff --git a/packages/cli/src/tako/takoSource.ts b/packages/cli/src/tako/takoSource.ts new file mode 100644 index 0000000..998fd4d --- /dev/null +++ b/packages/cli/src/tako/takoSource.ts @@ -0,0 +1,179 @@ +export type TakoMessageRole = "user" | "assistant"; + +export interface TakoMessage { + role: TakoMessageRole; + text: string; +} + +export interface ScreenFrame { + x: number; + y: number; + width: number; + height: number; +} + +interface ElementSection { + role: TakoMessageRole; + source: string; +} + +interface OpenElement { + role?: TakoMessageRole; + start: number; +} + +const COMPONENT_ROLES: Readonly> = { + "TikTokTakoImpl.TakoSendTextElementComponentV2": "user", + "TikTokTakoImpl.TakoTextElementComponentV2": "assistant", +}; + +function decodeXml(value: string): string { + return value.replace( + /&(#x[\da-f]+|#\d+|amp|apos|gt|lt|quot);/gi, + (_entity, encoded: string) => { + if (encoded.startsWith("#x")) { + return String.fromCodePoint(Number.parseInt(encoded.slice(2), 16)); + } + if (encoded.startsWith("#")) { + return String.fromCodePoint(Number.parseInt(encoded.slice(1), 10)); + } + return ( + { + amp: "&", + apos: "'", + gt: ">", + lt: "<", + quot: '"', + } as const + )[encoded.toLowerCase() as "amp" | "apos" | "gt" | "lt" | "quot"]; + } + ); +} + +function attribute(tag: string, name: string): string | undefined { + const match = tag.match(new RegExp(`\\b${name}="([^"]*)"`)); + return match?.[1] === undefined ? undefined : decodeXml(match[1]); +} + +function componentRole(tag: string): TakoMessageRole | undefined { + const name = attribute(tag, "name"); + return name === undefined ? undefined : COMPONENT_ROLES[name]; +} + +function findMessageSections(source: string): readonly ElementSection[] { + const tagPattern = /<\/?XCUIElementType\w+\b[^>]*>/g; + const initial: { + stack: readonly OpenElement[]; + sections: readonly ElementSection[]; + } = {stack: [], sections: []}; + + return [...source.matchAll(tagPattern)].reduce((state, match) => { + const tag = match[0]; + const index = match.index; + if (index === undefined) return state; + + if (tag.startsWith("")) { + return role + ? { + ...state, + sections: [...state.sections, {role, source: tag}], + } + : state; + } + return { + stack: [...state.stack, {role, start: index}], + sections: state.sections, + }; + }, initial).sections; +} + +function elementValues( + section: string, + elementType: string +): readonly string[] { + const values = [ + ...section.matchAll(new RegExp(`<${elementType}\\b[^>]*>`, "g")), + ] + .map(match => attribute(match[0], "value")?.trim()) + .filter( + (value): value is string => + Boolean(value) && !/^[\da-f]{8}$/i.test(value ?? "") + ); + return values.filter((value, index) => value !== values[index - 1]); +} + +function textViewValue(section: string): string | undefined { + const tag = section.match(/]*>/)?.[0]; + const value = tag === undefined ? undefined : attribute(tag, "value"); + const textViewText = value?.trim(); + const staticText = elementValues(section, "XCUIElementTypeStaticText").join( + "\n" + ); + if (textViewText && staticText) { + const prefix = textViewText.replace(/(?:\.\.\.|…)$/, "").trimEnd(); + return staticText.startsWith(prefix) ? staticText : undefined; + } + if (staticText) return staticText; + // XCTest caps some long text-view values at 512 characters. Returning that + // prefix as a complete benchmark response would silently lose content. + return textViewText && textViewText.length !== 512 ? textViewText : undefined; +} + +export function parseTakoMessages(source: string): readonly TakoMessage[] { + return findMessageSections(source).flatMap(section => { + const text = textViewValue(section.source); + return text === undefined ? [] : [{role: section.role, text}]; + }); +} + +export function isTakoSource(source: string): boolean { + return source.includes('name="TikTokTakoImpl.TakoRootComponent"'); +} + +export function findNamedFrame( + source: string, + elementName: string +): ScreenFrame | undefined { + return findNamedFrames(source, elementName)[0]; +} + +export function findNamedFrames( + source: string, + elementName: string +): readonly ScreenFrame[] { + return [...source.matchAll(/]*>/g)] + .map(match => match[0]) + .filter(tag => attribute(tag, "name") === elementName) + .flatMap(tag => { + const values = ["x", "y", "width", "height"].map(name => + Number(attribute(tag, name)) + ); + if (values.some(value => !Number.isFinite(value))) return []; + const [x, y, width, height] = values; + return x === undefined || + y === undefined || + width === undefined || + height === undefined + ? [] + : [{x, y, width, height}]; + }); +} + +export function frameCenter(frame: ScreenFrame): {x: number; y: number} { + return {x: frame.x + frame.width / 2, y: frame.y + frame.height / 2}; +} diff --git a/packages/cli/src/tako/takoWdaClient.ts b/packages/cli/src/tako/takoWdaClient.ts new file mode 100644 index 0000000..d7bfdde --- /dev/null +++ b/packages/cli/src/tako/takoWdaClient.ts @@ -0,0 +1,481 @@ +import * as fs from "node:fs/promises"; + +interface WdaEnvelope { + value: T; + sessionId?: string | null; +} + +interface WdaElementReference { + ELEMENT?: string; + "element-6066-11e4-a52e-4f735466cecf"?: string; +} + +interface WdaElementRect { + x: number; + y: number; + width: number; + height: number; +} + +interface WdaErrorValue { + error?: string; + message?: string; +} + +export type WdaElementType = + | "XCUIElementTypeAny" + | "XCUIElementTypeButton" + | "XCUIElementTypeImage" + | "XCUIElementTypeOther"; + +function sleep(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +export class WdaRequestError extends Error { + constructor( + message: string, + readonly timedOut: boolean + ) { + super(message); + this.name = "WdaRequestError"; + } +} + +export class TakoWdaClient { + private sessionId: string | undefined; + + constructor( + private readonly baseUrl: string, + private readonly bundleId: string + ) {} + + private async request( + method: "DELETE" | "GET" | "POST", + pathname: string, + body?: unknown, + timeoutMs = 20_000 + ): Promise { + const url = new URL(pathname, `${this.baseUrl.replace(/\/$/, "")}/`); + try { + const response = await fetch(url, { + method, + headers: + body === undefined ? undefined : {"content-type": "application/json"}, + body: body === undefined ? undefined : JSON.stringify(body), + signal: AbortSignal.timeout(timeoutMs), + }); + const responseText = await response.text(); + if (!response.ok) { + throw new Error( + `${method} ${url.pathname} returned ${response.status}: ${responseText}` + ); + } + if (!responseText.trim()) return undefined as T; + const envelope = JSON.parse(responseText) as WdaEnvelope; + const value = envelope.value as WdaErrorValue | T; + if ( + typeof value === "object" && + value !== null && + "error" in value && + typeof value.error === "string" + ) { + throw new Error(value.message ?? value.error); + } + return envelope.value; + } catch (error) { + const timedOut = error instanceof Error && error.name === "TimeoutError"; + throw new WdaRequestError( + `${method} ${url.pathname} failed: ${errorMessage(error)}`, + timedOut + ); + } + } + + private sessionPath(pathname: string): string { + if (!this.sessionId) throw new Error("WebDriverAgent session is not open."); + return `session/${this.sessionId}/${pathname.replace(/^\//, "")}`; + } + + private async focusedTextElement(): Promise { + const reference = await this.request( + "POST", + this.sessionPath("element"), + { + using: "class chain", + value: + '**/XCUIElementTypeOther[`name == "TikTokTakoImpl.TakoInputContainerViewComponent"`]/**/XCUIElementTypeTextView[-1]', + }, + 20_000 + ); + const elementId = + reference["element-6066-11e4-a52e-4f735466cecf"] ?? reference.ELEMENT; + if (!elementId) + throw new Error("Could not find Tako's focused text input."); + return elementId; + } + + private async clickElement(elementId: string): Promise { + try { + await this.request( + "POST", + this.sessionPath(`element/${encodeURIComponent(elementId)}/click`), + {}, + 10_000 + ); + } catch (error) { + if (!(error instanceof WdaRequestError) || !error.timedOut) throw error; + await sleep(1_000); + } + } + + async connect(): Promise { + const status = await this.request<{ready?: boolean}>( + "GET", + "status", + undefined, + 5_000 + ); + if (status.ready !== true) { + throw new Error("WebDriverAgent responded, but is not ready."); + } + const session = await this.request<{sessionId?: string}>( + "POST", + "session", + { + capabilities: { + alwaysMatch: { + bundleId: this.bundleId, + shouldTerminateApp: true, + shouldUseSingletonTestManager: false, + }, + }, + }, + 30_000 + ); + if (!session.sessionId) + throw new Error("WebDriverAgent did not return a session id."); + this.sessionId = session.sessionId; + + await this.request( + "POST", + this.sessionPath("appium/settings"), + { + settings: { + animationCoolOffTimeout: 0, + snapshotMaxDepth: 70, + useFirstMatch: true, + waitForIdleTimeout: 0, + }, + }, + 10_000 + ); + } + + async source(timeoutMs = 25_000): Promise { + return this.request( + "GET", + this.sessionPath("source"), + undefined, + timeoutMs + ); + } + + async ensureAppActive(): Promise { + // Activating the session's app is a cheap no-op when it is already in the + // foreground. It also avoids querying activeAppInfo, which can block while + // XCTest is servicing a large TikTok hierarchy snapshot. + await this.request( + "POST", + this.sessionPath("wda/apps/activate"), + {bundleId: this.bundleId}, + 15_000 + ); + await sleep(500); + } + + private async findFirstNamedElement( + names: readonly string[], + visibleOnly: boolean, + elementType: WdaElementType + ): Promise { + if (names.length === 0) return undefined; + const namePredicate = names + .map(name => `name == ${JSON.stringify(name)}`) + .join(" OR "); + try { + const reference = await this.request( + "POST", + this.sessionPath("element"), + { + using: "class chain", + value: `**/${elementType}[\`(${namePredicate})${visibleOnly ? " AND visible == 1" : ""}\`][1]`, + }, + 20_000 + ); + return ( + reference["element-6066-11e4-a52e-4f735466cecf"] ?? reference.ELEMENT + ); + } catch (error) { + if ( + error instanceof WdaRequestError && + error.message.toLocaleLowerCase().includes("no such element") + ) { + return undefined; + } + throw error; + } + } + + private async findFirstVisibleNamedDescendant( + ancestorName: string, + ancestorType: WdaElementType, + name: string, + elementType: WdaElementType + ): Promise { + const classChain = `**/${ancestorType}[\`name == ${JSON.stringify(ancestorName)}\`][1]/**/${elementType}[\`name == ${JSON.stringify(name)} AND visible == 1\`][1]`; + try { + const reference = await this.request( + "POST", + this.sessionPath("element"), + {using: "class chain", value: classChain}, + 20_000 + ); + const elementId = + reference["element-6066-11e4-a52e-4f735466cecf"] ?? reference.ELEMENT; + return elementId; + } catch (error) { + if ( + error instanceof WdaRequestError && + error.message.toLocaleLowerCase().includes("no such element") + ) { + return undefined; + } + throw error; + } + } + + async clickFirstVisibleNamedDescendant( + ancestorName: string, + ancestorType: WdaElementType, + name: string, + elementType: WdaElementType + ): Promise { + const elementId = await this.findFirstVisibleNamedDescendant( + ancestorName, + ancestorType, + name, + elementType + ); + if (!elementId) return false; + await this.clickElement(elementId); + return true; + } + + async hasNamedElement( + names: readonly string[], + visibleOnly = true, + elementType: WdaElementType = "XCUIElementTypeAny" + ): Promise { + return ( + (await this.findFirstNamedElement(names, visibleOnly, elementType)) !== + undefined + ); + } + + async clickFirstVisibleNamedElement( + names: readonly string[], + elementType: WdaElementType = "XCUIElementTypeAny" + ): Promise { + const elementId = await this.findFirstNamedElement( + names, + true, + elementType + ); + if (!elementId) return false; + await this.clickElement(elementId); + return true; + } + + async tap(x: number, y: number, timeoutMs = 10_000): Promise { + try { + await this.request( + "POST", + this.sessionPath("actions"), + { + actions: [ + { + type: "pointer", + id: "finger", + parameters: {pointerType: "touch"}, + actions: [ + { + type: "pointerMove", + duration: 0, + x: Math.round(x), + y: Math.round(y), + origin: "viewport", + }, + {type: "pointerDown", button: 0}, + {type: "pause", duration: 100}, + {type: "pointerUp", button: 0}, + ], + }, + ], + }, + timeoutMs + ); + } catch (error) { + if (!(error instanceof WdaRequestError) || !error.timedOut) throw error; + // TikTok's animated surfaces can keep XCTest's action response open + // briefly after the touch was delivered. The next source read verifies + // the resulting state, so a bounded action timeout is safe here. + await sleep(1_500); + } + } + + async setFocusedText(text: string): Promise { + const elementId = await this.focusedTextElement(); + await this.clickElement(elementId); + await this.request( + "POST", + this.sessionPath(`element/${encodeURIComponent(elementId)}/value`), + {text, value: Array.from(text)}, + 30_000 + ); + const value = await this.request( + "GET", + this.sessionPath( + `element/${encodeURIComponent(elementId)}/attribute/value` + ), + undefined, + 10_000 + ); + if (typeof value !== "string") { + throw new Error("Tako's focused text input did not expose a value."); + } + return value; + } + + async clickVisibleCopyButton(): Promise { + const elementId = await this.findFirstVisibleNamedDescendant( + "TikTokTakoImpl.TakoInteractionElementComponentV2", + "XCUIElementTypeOther", + "Copy", + "XCUIElementTypeButton" + ); + if (!elementId) { + throw new Error("Could not find Tako's visible Copy button."); + } + + // TikTok's animated transcript can leave XCTest's element-click command + // waiting indefinitely. Resolve the exact semantic button first, then ask + // XCTest for its current native frame and deliver one touch at that live + // frame. This is not an image-derived coordinate and cannot resolve to a + // suggested reply with a different accessibility identity. + const rect = await this.request( + "GET", + this.sessionPath(`element/${encodeURIComponent(elementId)}/rect`), + undefined, + 10_000 + ); + const values = [rect.x, rect.y, rect.width, rect.height]; + if ( + values.some(value => !Number.isFinite(value)) || + rect.width <= 0 || + rect.height <= 0 + ) { + throw new Error("Tako's Copy button returned an invalid native frame."); + } + await this.tap(rect.x + rect.width / 2, rect.y + rect.height / 2); + } + + async getPasteboardText(): Promise { + const encoded = await this.request( + "POST", + this.sessionPath("wda/getPasteboard"), + {contentType: "plaintext"}, + 15_000 + ); + return Buffer.from(encoded, "base64").toString("utf8"); + } + + async setPasteboardText(text: string): Promise { + await this.request( + "POST", + this.sessionPath("wda/setPasteboard"), + { + contentType: "plaintext", + content: Buffer.from(text, "utf8").toString("base64"), + }, + 15_000 + ); + } + + async saveScreenshot(filePath: string): Promise { + const base64 = await this.request( + "GET", + this.sessionPath("screenshot"), + undefined, + 30_000 + ); + await fs.writeFile(filePath, Buffer.from(base64, "base64")); + } + + async swipeUp(timeoutMs = 10_000): Promise { + try { + await this.request( + "POST", + this.sessionPath("actions"), + { + actions: [ + { + type: "pointer", + id: "finger", + parameters: {pointerType: "touch"}, + actions: [ + { + type: "pointerMove", + duration: 0, + x: 215, + y: 760, + origin: "viewport", + }, + {type: "pointerDown", button: 0}, + {type: "pause", duration: 100}, + { + type: "pointerMove", + duration: 500, + x: 215, + y: 310, + origin: "viewport", + }, + {type: "pointerUp", button: 0}, + ], + }, + ], + }, + timeoutMs + ); + } catch (error) { + if (!(error instanceof WdaRequestError) || !error.timedOut) throw error; + await sleep(1_500); + } + } + + async disconnect(): Promise { + if (!this.sessionId) return; + const sessionId = this.sessionId; + this.sessionId = undefined; + await this.request( + "DELETE", + `session/${sessionId}`, + undefined, + 5_000 + ).catch(() => {}); + } +} diff --git a/scripts/tako-copy-template.png.b64 b/scripts/tako-copy-template.png.b64 new file mode 100644 index 0000000..bffdc67 --- /dev/null +++ b/scripts/tako-copy-template.png.b64 @@ -0,0 +1 @@ +iVBORw0KGgoAAAANSUhEUgAAAGAAAABgCAAAAADH8yjkAAAAIGNIUk0AAHomAACAhAAA+gAAAIDoAAB1MAAA6mAAADqYAAAXcJy6UTwAAAACYktHRAD/h4/MvwAAAAlwSFlzAAALEgAACxIB0t1+/AAAAAd0SU1FB+oHCgAlAE5lftcAAAABb3JOVAHPoneaAAACLUlEQVRo3u2Yv2vbQBTHP5L8IzbFcUkaXAKFDoE4mTpkKISUQLJ0zz/Tv6V0arfM2YIp7lQKnRIPLRRKBlFcIifEamJZ6qQ2Tu6ku55Ef91bv1/po3dPenp3TkK54ZZ8fwuwAAuwAAuwAAuwAAuwAAsoLCr5lvDk7Go6u41wK83F+wUBLg6PY6HQ3u6qAJycHU7w8kyqbW4pAHJqEO/L78+bgTng6EuW2lMA5NRgALDbnfOc2cQmpwc+BH7HNIMhsLJxp+LcuKreeZrKZoAQaAuVNsDYGDABqkKlBhAZA2KpxU1lMwCAI5cUjgn+/mb3W7ppfDQYjqMfBez3AcertR48WigEEOwL+kMSRWP/3ZPHBQAuXo1k5riHNuF2DQ5HGfbXX40zCI8BVu5WZ9HJNPw0gvj9jingJAZ2NwTWy+cj+Gy8ROcAqyJr/WEqGwEuARpC71wqGwGmAJ7Q66WyESABWX9zudbeMnzZANXIyLQYQAhQLxEwAGjl+xRGx9noA0k8Of0IuMslAdJYa+T7Tf4H8yptQzuDn7G01ywN4FYb91bXlbLXBjzTfZZfyuD/BpjNcaoA8cwpH1R1AFWAK6FXPmrrAJoAgdAbIPvX6QAWAQ78bzdWKZmcv/2Qyjpx60PrtAPwX8j8SnvjzAzYzrIvrZsDupty9/ye9lsk2ukPeoH4YdZ2VPpnPgD84TiKryuO49Vay9qvEPlnFcbxDzY7C7AAC7AAC/gTAd8BJoGDdXr1MwIAAACEZVhJZk1NACoAAAAIAAUBEgADAAAAAQABAAABGgAFAAAAAQAAAEoBGwAFAAAAAQAAAFIBKAADAAAAAQACAACHaQAEAAAAAQAAAFoAAAAAAAAASAAAAAEAAABIAAAAAQADoAEAAwAAAAEAAQAAoAIABAAAAAEAAAUKoAMABAAAAAEAAArsAAAAACq3pQgAAAAldEVYdGRhdGU6Y3JlYXRlADIwMjYtMDctMDlUMjM6MTU6MDUrMDA6MDDAEyqnAAAAJXRFWHRkYXRlOm1vZGlmeQAyMDI2LTA3LTA5VDIzOjE1OjA1KzAwOjAwsU6SGwAAACh0RVh0ZGF0ZTp0aW1lc3RhbXAAMjAyNi0wNy0xMFQwMDozNzowMCswMDowMO6DmGEAAAARdEVYdGV4aWY6Q29sb3JTcGFjZQAxD5sCSQAAABJ0RVh0ZXhpZjpFeGlmT2Zmc2V0ADkwWYzemwAAABl0RVh0ZXhpZjpQaXhlbFhEaW1lbnNpb24AMTI5MBXYn6MAAAAZdEVYdGV4aWY6UGl4ZWxZRGltZW5zaW9uADI3OTZRPox7AAAAAElFTkSuQmCC diff --git a/scripts/tako-new-chat-template.png.b64 b/scripts/tako-new-chat-template.png.b64 new file mode 100644 index 0000000..f73c28e --- /dev/null +++ b/scripts/tako-new-chat-template.png.b64 @@ -0,0 +1 @@ +iVBORw0KGgoAAAANSUhEUgAAAGAAAABgCAAAAADH8yjkAAAAIGNIUk0AAHomAACAhAAA+gAAAIDoAAB1MAAA6mAAADqYAAAXcJy6UTwAAAACYktHRAD/h4/MvwAAAAlwSFlzAAALEgAACxIB0t1+/AAAAAd0SU1FB+oHCgAlFSO4mjwAAAABb3JOVAHPoneaAAAChklEQVRo3u2ZsYsTQRSHf7vJxVUwJzZ6RJQDOXsLvSsMVlEQrK5btDVN0Eru/CuuOgsb65SCXNrDwjqNhaTzFruAuJjAqmMxGw3se7PvTbjmnFeFzJvvm7wkk5mXyOB0Iz5lfhAEQRAEQRAEQY0ge/10+8alpBktx/lXXHrBDRg6srRBpid0ev4w3qNHGMGwrVpQ3gXwQiE4iKAR5F2AM5AThjyfKlF+rxzcEwqyizx/v5o+W/ARF9XRJkF5+R0A0Hr26OblC80GaiK5/aF81KNoVeeJRXbGRhrPLaqby0p0aNcv55cGkk8JngAABiJyvvtpYaD5lOAOAGAk4ndxpTQwfEpwHQAwkfGxMMyYHEKwDgCYCvl/DXLBOQDAXMqvMRACu8H+EvOBXZ3AuctSfO79XUkg5fsKxHxPgZzvJ1DwvQQavo9AxfcQ6Ph6gZKvFmj5WoGarxTo+TqBB18l8OFrBEXPg68R9J38/YQ+lskFR+71JwB5sBQLik13fbjXLRa8ran/yoLtkt/jzj8rCr6UJ/p+YehYVfAGALB5ZLjgBNSBm4pjILrbT6Xp/0I6Ywf3H1xT0wFE1casrba2YcvNOss3/f9NwH4mYu6p3zqBTSeuvIRgDYCje0KHTSe+VYQgAQD80AlmS1PrBOUdTSeYLk2tE1wFAEx0Apu+IRLcAgC81wls+hYxUt1gPVoJZtwCAByKfg/0zRAz7gAAGicigUnta2sNRpPpvOY6+3M+nYwGdv1IjUyQteEV7UwocLXUHBENjVTgagry/AMjFzjammx9hkYjYBuzTDTSjAFF7D6bvfv4+eu3eeHeVuO1ZH1ja+dxh61c+Dc2CIIgCIIgCM6E4A86SXYv/YHRSQAAAIRlWElmTU0AKgAAAAgABQESAAMAAAABAAEAAAEaAAUAAAABAAAASgEbAAUAAAABAAAAUgEoAAMAAAABAAIAAIdpAAQAAAABAAAAWgAAAAAAAABIAAAAAQAAAEgAAAABAAOgAQADAAAAAQABAACgAgAEAAAAAQAABQqgAwAEAAAAAQAACuwAAAAAKrelCAAAACV0RVh0ZGF0ZTpjcmVhdGUAMjAyNi0wNy0wOVQyMzoxNTowNSswMDowMMATKqcAAAAldEVYdGRhdGU6bW9kaWZ5ADIwMjYtMDctMDlUMjM6MTU6MDUrMDA6MDCxTpIbAAAAKHRFWHRkYXRlOnRpbWVzdGFtcAAyMDI2LTA3LTEwVDAwOjM3OjIxKzAwOjAwCtGUqAAAABF0RVh0ZXhpZjpDb2xvclNwYWNlADEPmwJJAAAAEnRFWHRleGlmOkV4aWZPZmZzZXQAOTBZjN6bAAAAGXRFWHRleGlmOlBpeGVsWERpbWVuc2lvbgAxMjkwFdifowAAABl0RVh0ZXhpZjpQaXhlbFlEaW1lbnNpb24AMjc5NlE+jHsAAAAASUVORK5CYII= diff --git a/scripts/tako-ocr.swift b/scripts/tako-ocr.swift new file mode 100644 index 0000000..df4be26 --- /dev/null +++ b/scripts/tako-ocr.swift @@ -0,0 +1,63 @@ +import CoreGraphics +import Foundation +import ImageIO +import Vision + +func fail(_ message: String) -> Never { + FileHandle.standardError.write(Data("\(message)\n".utf8)) + exit(1) +} + +guard CommandLine.arguments.count == 6 else { + fail("Usage: tako-ocr.swift ") +} + +let arguments = CommandLine.arguments +let imageUrl = URL(fileURLWithPath: arguments[1]) +guard + let x = Double(arguments[2]), + let y = Double(arguments[3]), + let width = Double(arguments[4]), + let height = Double(arguments[5]), + let source = CGImageSourceCreateWithURL(imageUrl as CFURL, nil), + let image = CGImageSourceCreateImageAtIndex(source, 0, nil) +else { + fail("Could not read the image or crop coordinates.") +} + +let logicalWidth = 430.0 +let logicalHeight = 932.0 +let request = VNRecognizeTextRequest() +request.recognitionLevel = .accurate +request.usesLanguageCorrection = true +request.minimumTextHeight = 0.012 +request.regionOfInterest = CGRect( + x: x / logicalWidth, + y: 1.0 - ((y + height) / logicalHeight), + width: width / logicalWidth, + height: height / logicalHeight +) + +do { + try VNImageRequestHandler(cgImage: image, orientation: .up).perform([request]) +} catch { + fail("Vision OCR failed: \(error)") +} + +let lines = (request.results ?? []) + .sorted { left, right in + if abs(left.boundingBox.maxY - right.boundingBox.maxY) > 0.01 { + return left.boundingBox.maxY > right.boundingBox.maxY + } + return left.boundingBox.minX < right.boundingBox.minX + } + .compactMap { $0.topCandidates(1).first?.string } + +guard + let json = try? JSONSerialization.data(withJSONObject: lines), + let output = String(data: json, encoding: .utf8) +else { + fail("Could not encode OCR output.") +} + +print(output) diff --git a/scripts/tako-ui-match.py b/scripts/tako-ui-match.py new file mode 100644 index 0000000..62d1c56 --- /dev/null +++ b/scripts/tako-ui-match.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 + +import json +import sys + +import numpy as np +from PIL import Image + + +def fail(message: str) -> None: + print(message, file=sys.stderr) + raise SystemExit(1) + + +if len(sys.argv) != 6: + fail( + "Usage: tako-ui-match.py