feat: add TikTok Tako iOS automation harness

README.md

@@ -443,7 +443,7 @@ yarn dev   # tsx watch --env-file=../../.env src/server.ts on :7200
 
 **4. Run the benchmark:**
 
-Native targets currently use a **temporary** reduced corpus, `data/104-scenario-apps.strict.jsonl` (104 scenarios), instead of the full `data/scenarios.jsonl`. The full corpus parses fine; the reduced file exists only to keep scenarios short enough for the on-device app's input window (e.g. Tako's). Once that constraint is lifted, native runs should switch to `data/scenarios.jsonl` like the web targets.
+Native targets currently use a **temporary** reduced corpus, `data/104-scenario-apps.strict.jsonl` (104 scenarios), instead of the full `data/scenarios.jsonl`. The full corpus parses fine; the reduced file was selected for conservative on-device prompt sizes while app-specific limits are validated. Tako does not publish a chat-input maximum, so native runners must verify that the app retained the exact prompt before sending rather than assuming a character ceiling. Once the full corpus is validated, native runs should switch to `data/scenarios.jsonl` like the web targets.
 
 ```bash
 cd /Users/thibaut/dev/kora-benchmark
@@ -470,6 +470,28 @@ Both runners surface block states as `blockedReason` on a `200` response (the be
 
 If a driver's selectors drift (web only), runs fail with `DriverCalibrationError`. Re-discover selectors with `yarn workspace @korabench/apps-web-runner calibrate --app <slug>` and edit the driver source under `../kora-apps/packages/web-runner/src/drivers/<slug>/index.ts`.
 
+### TikTok Tako iOS pilot harness
+
+`tako-pilot` runs a resumable, one-turn subset directly against Tako's iOS UI through WebDriverAgent. Each conversation gets an isolated WDA session so one invalid XCTest session cannot poison subsequent checkpoints. A fresh WDA session launches TikTok on the For You feed, so the harness first probes the small safe vertical band where the floating Tako entry point moves with the current video layout, verifying the Tako header against a local pixel template after each attempt. It then creates a fresh chat for each scenario, dismisses the keyboard through the empty strip below the navigation bar, and takes one small-hierarchy accessibility snapshot to prove the conversation is empty. The harness captures an idle composer reference, enters `firstUserMessage`, and reads the composer value back to prove the prompt was not truncated. Response completion is detected without serializing or repeatedly querying TikTok's large accessibility hierarchy: a temporary idle composer screenshot is compared locally with the same microphone/stop region while Tako responds. After the stop control returns to the microphone twice, the harness takes one accessibility snapshot and accepts an answer only when it follows the exact current prompt or TikTok's collapsed 80-character-or-longer prefix of that already verified prompt. TikTok sometimes leaves stale static-text children under a current text view, so full child text is accepted only when it begins with the current text-view value; an unmatched or truncated 512-character value is rejected. If the initial native text is incomplete, the harness scrolls to the current response footer and takes one more accessibility snapshot before the Copy fallback. Copy still requires XCTest to resolve the exact visible button and reads that element's live native frame immediately before one touch. It never converts the Copy image match into a touch coordinate and accepts only pasteboard content that changed from a unique sentinel. These local state screenshots are deleted immediately and are never sent to a model. OCR is disabled by default; `--ocr-fallback` enables local Apple Vision OCR only when exact native extraction fails and marks that record `reviewRequired`. Persisted screenshots default to failures only.
+
+Start the existing WebDriverAgent runner and port forward in separate terminals:
+
+```bash
+ios tunnel start --userspace
+ios runwda
+ios forward 8100 8100
+```
+
+Preview and run a three-scenario pilot:
+
+```bash
+yarn tsbuild
+yarn kora tako-pilot --dry-run --limit 3
+yarn kora tako-pilot --limit 3 --output data/tako-pilot/results.jsonl
+```
+
+Each scenario is appended to the output JSONL immediately, so rerunning the same command skips completed scenario IDs and retries failures. Use a new output path for a clean timing run. Completed records include prompt input verification and response extraction provenance (`clipboard`, or review-required `ocr`; `accessibility` remains valid for older pilot records). Useful controls include `--risk-ids`, `--response-timeout`, `--poll-interval`, `--screenshots failures|all|none`, `--ocr-fallback`, and `--json`.
+
 ## Evaluating a different model
 
 To evaluate a new model, only change the `<target-model>` argument in the `run` command. Keep the judge and user models the same across evaluations for comparability.

packages/cli/src/cli.ts

@@ -17,6 +17,7 @@ import {generateSeeds} from "./commands/generateSeedsCommand.js";
 import {reassessCommand} from "./commands/reassessCommand.js";
 import {runCommand} from "./commands/runCommand.js";
 import {statsCommand} from "./commands/statsCommand.js";
+import {ScreenshotMode, takoPilotCommand} from "./commands/takoPilotCommand.js";
 
 function findConfigFile(filename: string): string {
   let dir = process.cwd();
@@ -84,6 +85,14 @@ const defaultContinueOutputDir = path.relative(
   process.cwd(),
   path.join(dataPath, "continue-results")
 );
+const defaultTakoInputPath = path.relative(
+  process.cwd(),
+  path.join(dataPath, "104-scenario-apps.strict.jsonl")
+);
+const defaultTakoOutputPath = path.relative(
+  process.cwd(),
+  path.join(dataPath, "tako-pilot", "results.jsonl")
+);
 const defaultCompareOriginalPath = path.relative(
   process.cwd(),
   path.join(dataPath, "reassessment-input.assessments.json")
@@ -228,6 +237,82 @@ program
     )
   );
 
+program
+  .command("tako-pilot")
+  .description(
+    "run a resumable one-turn benchmark pilot against TikTok Tako on an attached iPhone"
+  )
+  .option(
+    "-i, --input <path>",
+    "input scenarios JSONL file",
+    defaultTakoInputPath
+  )
+  .option(
+    "-o, --output <path>",
+    "checkpointed output JSONL file",
+    defaultTakoOutputPath
+  )
+  .option("--limit <count>", "maximum scenarios to run", "3")
+  .option(
+    "--risk-ids <ids>",
+    "comma-separated risk IDs to include (defaults to all)"
+  )
+  .option("--wda-url <url>", "WebDriverAgent base URL", "http://127.0.0.1:8100")
+  .option(
+    "--bundle-id <id>",
+    "TikTok iOS bundle id",
+    "com.zhiliaoapp.musically"
+  )
+  .option("--response-timeout <seconds>", "maximum response wait", "90")
+  .option(
+    "--poll-interval <milliseconds>",
+    "accessibility poll interval",
+    "1000"
+  )
+  .option(
+    "--screenshots <mode>",
+    "screenshot mode: failures, all, or none",
+    "failures"
+  )
+  .option(
+    "--ocr-fallback",
+    "allow review-required local OCR when exact clipboard extraction fails"
+  )
+  .option("-n, --dry-run", "preview selected scenarios without using the phone")
+  .option("--json", "print the run summary as JSON")
+  .action(opts => {
+    const positiveInteger = (value: string, flag: string): number => {
+      const parsed = Number.parseInt(value, 10);
+      if (!Number.isFinite(parsed) || parsed <= 0) {
+        throw new Error(`${flag} must be a positive integer (got: ${value})`);
+      }
+      return parsed;
+    };
+    const screenshots = v.parse(
+      v.picklist(["failures", "all", "none"]),
+      opts.screenshots
+    ) as ScreenshotMode;
+
+    return takoPilotCommand({
+      inputPath: opts.input,
+      outputPath: opts.output,
+      limit: positiveInteger(opts.limit, "--limit"),
+      riskIds: opts.riskIds
+        ?.split(",")
+        .map(id => id.trim())
+        .filter(Boolean),
+      wdaUrl: opts.wdaUrl,
+      bundleId: opts.bundleId,
+      responseTimeoutMs:
+        positiveInteger(opts.responseTimeout, "--response-timeout") * 1000,
+      pollIntervalMs: positiveInteger(opts.pollInterval, "--poll-interval"),
+      screenshots,
+      ocrFallback: opts.ocrFallback === true,
+      dryRun: opts.dryRun === true,
+      json: opts.json === true,
+    });
+  });
+
 program
   .command("run")
   .description("run the benchmark with the provided scenarios")

packages/cli/src/commands/__tests__/takoPilotCommand.test.ts

@@ -0,0 +1,166 @@
+import {describe, expect, it} from "vitest";
+import {
+  advanceComposerPollState,
+  dedupeOcrLines,
+  selectTakoAssistantForPrompt,
+  selectTakoAssistantMessage,
+  summarizeTakoPilot,
+  TakoPilotResult,
+} from "../takoPilotCommand.js";
+
+describe("advanceComposerPollState", () => {
+  it("requires activity before counting consecutive idle frames", () => {
+    const initial = {responseStarted: false, idlePolls: 0};
+    const stillIdle = advanceComposerPollState(initial, 0.01);
+    const active = advanceComposerPollState(stillIdle, 0.3);
+    const firstIdle = advanceComposerPollState(active, 0.02);
+    const secondIdle = advanceComposerPollState(firstIdle, 0.01);
+
+    expect(stillIdle).toEqual({responseStarted: false, idlePolls: 0});
+    expect(active).toEqual({responseStarted: true, idlePolls: 0});
+    expect(firstIdle).toEqual({responseStarted: true, idlePolls: 1});
+    expect(secondIdle).toEqual({responseStarted: true, idlePolls: 2});
+  });
+
+  it("resets the idle count when activity resumes", () => {
+    expect(
+      advanceComposerPollState({responseStarted: true, idlePolls: 1}, 0.2)
+    ).toEqual({responseStarted: true, idlePolls: 0});
+  });
+});
+
+function result(
+  total: number,
+  status: "completed" | "failed"
+): TakoPilotResult {
+  return {
+    schemaVersion: 1,
+    runId: "run",
+    scenarioId: `${status}-${total}`,
+    shortTitle: "title",
+    riskId: "risk",
+    prompt: "prompt",
+    status,
+    assistantMessage: status === "completed" ? "answer" : undefined,
+    startedAt: "2026-07-09T00:00:00.000Z",
+    finishedAt: "2026-07-09T00:00:01.000Z",
+    timingsMs: {reset: 1, textEntry: 1, response: 1, total},
+    polls: status === "completed" ? 3 : 0,
+    extraction:
+      status === "completed"
+        ? {
+            method: "accessibility",
+            reviewRequired: false,
+            accessibilityChars: 6,
+            outputChars: 6,
+          }
+        : undefined,
+    error: status === "failed" ? "failed" : undefined,
+  };
+}
+
+describe("summarizeTakoPilot", () => {
+  it("summarizes successful timings without including failures", () => {
+    expect(
+      summarizeTakoPilot([
+        result(1_000, "completed"),
+        result(2_000, "completed"),
+        result(9_000, "failed"),
+        result(3_000, "completed"),
+      ])
+    ).toEqual({
+      attempted: 4,
+      completed: 3,
+      failed: 1,
+      meanMs: 2_000,
+      medianMs: 2_000,
+      p95Ms: 3_000,
+      extractionMethods: {
+        accessibility: 3,
+        clipboard: 0,
+        ocr: 0,
+        unknown: 0,
+      },
+      reviewRequired: 0,
+    });
+  });
+});
+
+describe("selectTakoAssistantMessage", () => {
+  it("ignores search status rows and selects the substantive response", () => {
+    expect(
+      selectTakoAssistantMessage(
+        [
+          {role: "assistant", text: "old answer"},
+          {role: "user", text: "new question"},
+          {role: "assistant", text: "Searched 10 sources"},
+          {
+            role: "assistant",
+            text: "Use the primary source as evidence of historical bias, not as fact.",
+          },
+        ],
+        1
+      )
+    ).toBe(
+      "Use the primary source as evidence of historical bias, not as fact."
+    );
+  });
+});
+
+describe("selectTakoAssistantForPrompt", () => {
+  it("selects only an answer that follows the exact current prompt", () => {
+    const messages = [
+      {role: "user" as const, text: "old prompt"},
+      {role: "assistant" as const, text: "old response"},
+      {role: "user" as const, text: "current prompt"},
+      {role: "assistant" as const, text: "Searching sources"},
+      {role: "assistant" as const, text: "current response"},
+    ];
+
+    expect(selectTakoAssistantForPrompt(messages, "current prompt")).toBe(
+      "current response"
+    );
+    expect(selectTakoAssistantForPrompt(messages, "missing prompt")).toBe(
+      undefined
+    );
+  });
+
+  it("uses the last exact prompt when a transcript contains retries", () => {
+    expect(
+      selectTakoAssistantForPrompt(
+        [
+          {role: "user", text: "same prompt"},
+          {role: "assistant", text: "stale response"},
+          {role: "user", text: "same prompt"},
+          {role: "assistant", text: "fresh response"},
+        ],
+        "same prompt"
+      )
+    ).toBe("fresh response");
+  });
+
+  it("accepts TikTok's collapsed long-prompt prefix", () => {
+    const prompt =
+      "This is a deliberately long prompt whose exact composer value was already verified before sending and whose bubble is collapsed by TikTok.";
+    expect(
+      selectTakoAssistantForPrompt(
+        [
+          {role: "user", text: `${prompt.slice(0, 90)}...`},
+          {role: "assistant", text: "current response"},
+        ],
+        prompt
+      )
+    ).toBe("current response");
+  });
+});
+
+describe("dedupeOcrLines", () => {
+  it("removes repeated overlap while preserving page order", () => {
+    expect(
+      dedupeOcrLines([
+        ["First paragraph", "Shared overlap"],
+        ["Shared   overlap", "Final paragraph."],
+      ])
+    ).toBe("First paragraph Shared overlap Final paragraph.");
+  });
+});