diff --git a/packages/conversation-ui-core/src/composer-input-parts.ts b/packages/conversation-ui-core/src/composer-input-parts.ts index d22cab13a..efd45d1be 100644 --- a/packages/conversation-ui-core/src/composer-input-parts.ts +++ b/packages/conversation-ui-core/src/composer-input-parts.ts @@ -1,4 +1,5 @@ import { parseLocalFileReference } from "./message-body.js"; +import { trimBoundaryCharacters } from "./string-boundaries.js"; export type ComposerInputPart = | { readonly type: "text"; readonly text: string } @@ -254,7 +255,7 @@ function mergeAdjacentTextParts(parts: readonly ComposerInputPart[]): readonly C } function normalizeLabel(value: string): string { - return value.toLowerCase().replace(/[^a-z0-9_.-]+/g, "-").replace(/^-+|-+$/g, ""); + return trimBoundaryCharacters(value.toLowerCase().replace(/[^a-z0-9_.-]+/g, "-"), "-"); } function readKnownMentionAtStart( @@ -319,7 +320,7 @@ function readUrlAtStart( } function trimUrlToken(value: string): string { - let next = trimNaturalLanguageSuffix(value).replace(/[.,;:!?,。!?;:、]+$/u, ""); + let next = trimBoundaryCharacters(trimNaturalLanguageSuffix(value), ".,;:!?,。!?;:、", "end"); while (next.endsWith(")") || next.endsWith("]")) { const opener = next.endsWith(")") ? "(" : "["; const closer = next.endsWith(")") ? ")" : "]"; diff --git a/packages/conversation-ui-core/src/message-body.ts b/packages/conversation-ui-core/src/message-body.ts index 68a341452..84d21772a 100644 --- a/packages/conversation-ui-core/src/message-body.ts +++ b/packages/conversation-ui-core/src/message-body.ts @@ -2,6 +2,7 @@ import { unified } from "unified"; import remarkParse from "remark-parse"; import remarkGfm from "remark-gfm"; import remarkMath from "remark-math"; +import { trimBoundaryCharacters } from "./string-boundaries.js"; export interface LocalFileReference { readonly raw: string; @@ -521,11 +522,11 @@ function stripWrappingQuotes(value: string): string { } function trimTrailingUrlPunctuation(value: string): string { - return value.replace(/[),.;!?]+$/g, ""); + return trimBoundaryCharacters(value, "),.;!?", "end"); } function trimTrailingPathPunctuation(value: string): string { - return value.replace(/[),.;!?]+$/g, ""); + return trimBoundaryCharacters(value, "),.;!?", "end"); } function normalizePath(value: string): string { diff --git a/packages/conversation-ui-core/src/string-boundaries.ts b/packages/conversation-ui-core/src/string-boundaries.ts new file mode 100644 index 000000000..6456f141e --- /dev/null +++ b/packages/conversation-ui-core/src/string-boundaries.ts @@ -0,0 +1,74 @@ +export function trimBoundaryCharacters( + value: string, + characters: string, + side: "start" | "end" | "both" = "both", +): string { + let start = 0; + let end = value.length; + if (side !== "end") { + while (start < end && characters.includes(value[start] ?? "")) start += 1; + } + if (side !== "start") { + while (end > start && characters.includes(value[end - 1] ?? "")) end -= 1; + } + return start === 0 && end === value.length ? value : value.slice(start, end); +} + +export function extractWrappedSection( + value: string, + labels: readonly string[], + boundaryLabels: readonly string[], +): string | undefined { + const searchValue = asciiLowercase(value); + let selected: { readonly labelIndex: number; readonly start: number } | undefined; + for (const label of labels) { + const searchLabel = asciiLowercase(label); + let labelIndex = searchValue.indexOf(searchLabel); + while (labelIndex !== -1) { + let start = labelIndex + label.length; + while (start < value.length && isWhitespace(value[start])) start += 1; + if ("::".includes(value[start] ?? "")) { + start += 1; + while (start < value.length && isWhitespace(value[start])) start += 1; + if (!selected || labelIndex < selected.labelIndex) selected = { labelIndex, start }; + break; + } + labelIndex = searchValue.indexOf(searchLabel, labelIndex + 1); + } + } + if (!selected) return undefined; + let end = value.length; + for (const boundary of boundaryLabels) { + const searchBoundary = asciiLowercase(boundary); + let boundaryIndex = searchValue.indexOf(searchBoundary, selected.start); + while (boundaryIndex !== -1) { + const before = value[boundaryIndex - 1]; + const after = value[boundaryIndex + boundary.length]; + if (isWhitespace(before) && !isWordCharacter(after)) { + end = Math.min(end, boundaryIndex); + break; + } + boundaryIndex = searchValue.indexOf(searchBoundary, boundaryIndex + 1); + } + } + const section = value.slice(selected.start, end).trim(); + return section || undefined; +} + +function asciiLowercase(value: string): string { + const result: string[] = []; + for (let index = 0; index < value.length; index += 1) { + const character = value[index] ?? ""; + const code = character.charCodeAt(0); + result.push(code >= 65 && code <= 90 ? String.fromCharCode(code + 32) : character); + } + return result.join(""); +} + +function isWhitespace(value: string | undefined): boolean { + return value !== undefined && value.trim() === ""; +} + +function isWordCharacter(value: string | undefined): boolean { + return value !== undefined && /[A-Za-z0-9_]/u.test(value); +} diff --git a/packages/conversation-ui-core/src/terminal-report-view-model.ts b/packages/conversation-ui-core/src/terminal-report-view-model.ts index 7a579e0a2..570f5eb9d 100644 --- a/packages/conversation-ui-core/src/terminal-report-view-model.ts +++ b/packages/conversation-ui-core/src/terminal-report-view-model.ts @@ -14,6 +14,7 @@ import { userVisibleTerminalReportChanges, } from "./terminal-report-visibility.js"; import { projectVisibleTerminalReportContent } from "./terminal-report-presentation.js"; +import { extractWrappedSection, trimBoundaryCharacters } from "./string-boundaries.js"; interface TerminalReportSourceRefInput { readonly kind: string; @@ -300,10 +301,17 @@ function uniqueSourceRefs(refs: readonly TerminalReportSourceRefInput[]): Termin */ function compactTerminalReportTitle(value: string): string { const normalized = value.replace(/\s+/gu, " ").trim(); - const wrapped = normalized.match( - /(?:原始请求|Original request)\s*[::]\s*(.+?)(?=\s+(?:执行任务|已物化的执行操作|建议协作步骤|Requested objective|No literal file content|Materialized operations|Participant operation ownership|Permission boundary)\b|$)/iu, - )?.[1]?.trim(); - const candidate = (wrapped || normalized).replace(/^\*+|\*+$/gu, "").trim(); + const wrapped = extractWrappedSection(normalized, ["原始请求", "Original request"], [ + "执行任务", + "已物化的执行操作", + "建议协作步骤", + "Requested objective", + "No literal file content", + "Materialized operations", + "Participant operation ownership", + "Permission boundary", + ]); + const candidate = trimBoundaryCharacters(wrapped || normalized, "*").trim(); return candidate.length > 180 ? `${candidate.slice(0, 177).trimEnd()}...` : candidate; } diff --git a/packages/conversation-ui-core/test/composer-policy.test.mjs b/packages/conversation-ui-core/test/composer-policy.test.mjs index a73142c3d..54e456b36 100644 --- a/packages/conversation-ui-core/test/composer-policy.test.mjs +++ b/packages/conversation-ui-core/test/composer-policy.test.mjs @@ -114,6 +114,18 @@ test("raw token removal trims duplicate whitespace at removal boundary", () => { assert.equal(removeComposerInputRawToken("Run /review now", "/review"), "Run now"); }); +test("composer parsing handles long boundary runs without changing tokens", () => { + const punctuation = "!".repeat(100_000); + const parts = parseComposerInputParts(`https://example.com/docs${punctuation}`); + assert.equal(parts[0]?.type, "url_ref"); + assert.equal(parts[0]?.raw, "https://example.com/docs"); + + const paddedId = `${"-".repeat(100_000)}review${"-".repeat(100_000)}`; + const command = parseComposerInputParts("/review", { slashCommands: [{ id: paddedId, label: "Review" }] }); + assert.equal(command[0]?.type, "slash_command"); + assert.equal(command[0]?.commandId, paddedId); +}); + test("permission confirmation is derived from explicit option metadata", () => { assert.equal(conversationPermissionPresetRequiresConfirmation("full_access"), true); assert.equal(conversationPermissionPresetRequiresConfirmation("read_only"), false); diff --git a/packages/conversation-ui-core/test/message-body.test.mjs b/packages/conversation-ui-core/test/message-body.test.mjs index f26a3e682..7f39f7018 100644 --- a/packages/conversation-ui-core/test/message-body.test.mjs +++ b/packages/conversation-ui-core/test/message-body.test.mjs @@ -144,3 +144,12 @@ test("message body collects unique file references from nested structured blocks "docs/design/conversation-shared-capabilities.md", ]); }); + +test("message tokenization trims long punctuation runs in linear boundary helpers", () => { + const punctuation = "!".repeat(100_000); + const tokens = splitInlineTokens(`https://example.com/docs${punctuation} src/index.ts${punctuation}`, "/workspace/ling-stack"); + const link = tokens.find((token) => token.kind === "web_link"); + const file = tokens.find((token) => token.kind === "local_file"); + assert.equal(link?.kind === "web_link" ? link.href : undefined, "https://example.com/docs"); + assert.equal(file?.kind === "local_file" ? file.reference.displayPath : undefined, "src/index.ts"); +}); diff --git a/packages/conversation-ui-core/test/terminal-report-view-model.test.mjs b/packages/conversation-ui-core/test/terminal-report-view-model.test.mjs index d79060a2a..957e9bd96 100644 --- a/packages/conversation-ui-core/test/terminal-report-view-model.test.mjs +++ b/packages/conversation-ui-core/test/terminal-report-view-model.test.mjs @@ -140,6 +140,20 @@ test("terminal report projection keeps Markdown summaries and bounds model title assert.match(participant?.description ?? "", /```ts\nconst ok = true;/u); }); +test("terminal report title extraction handles long protocol whitespace", () => { + const title = `Original request${" ".repeat(100_000)}: concise result Requested objective ignored wrapper`; + const report = terminalReportToViewModel({ + id: "terminal_report_long_wrapper", + status: "committed", + language: "en-US", + outcome: { status: "succeeded", title, summary: "done", corrections: [], sourceRefs: [] }, + changes: [], verification: [], participants: [], artifacts: [], risks: [], references: [], + evidenceFingerprint: "fingerprint_long_wrapper", + committedAt: "2026-10-21T00:00:00.000Z", + }); + assert.equal(report.rows.find((row) => row.reportSection === "outcome")?.title, "concise result"); +}); + test("terminal report artifact workspace paths require one trusted changed-file source ref", () => { const report = terminalReportToViewModel({ id: "terminal_report_workspace_files", diff --git a/packages/core/src/channel-team-snapshot-harness.ts b/packages/core/src/channel-team-snapshot-harness.ts index e92423968..a4368f34b 100644 --- a/packages/core/src/channel-team-snapshot-harness.ts +++ b/packages/core/src/channel-team-snapshot-harness.ts @@ -111,7 +111,7 @@ function inferRiskReview(input: string): Parameters value.indexOf(target, targetStart) !== -1)) { + return true; + } + } + return false; +} + function diffValue(path: string, before: unknown, after: unknown): readonly ChannelTeamSnapshotDiffEntry[] { if (JSON.stringify(before) === JSON.stringify(after)) { return []; diff --git a/packages/core/src/composer-slash-intents.ts b/packages/core/src/composer-slash-intents.ts index bdebbf2b7..42419c123 100644 --- a/packages/core/src/composer-slash-intents.ts +++ b/packages/core/src/composer-slash-intents.ts @@ -1,3 +1,5 @@ +import { trimBoundaryCharacters } from "./string-boundaries.js"; + export type ComposerSlashIntent = | { readonly kind: "none" } | { @@ -314,7 +316,7 @@ function isSessionCommandOption(command: ComposerSlashCommandOption): boolean { } function normalizeLabel(value: string): string { - return value.toLowerCase().replace(/[^a-z0-9_.-]+/g, "-").replace(/^-+|-+$/g, ""); + return trimBoundaryCharacters(value.toLowerCase().replace(/[^a-z0-9_.-]+/g, "-"), "-"); } function teammateMentionAliases(teammate: ComposerSlashMentionOption): readonly string[] { diff --git a/packages/core/src/external-work-actions.ts b/packages/core/src/external-work-actions.ts index 0837cd5c3..3d69a94c1 100644 --- a/packages/core/src/external-work-actions.ts +++ b/packages/core/src/external-work-actions.ts @@ -3,6 +3,7 @@ import { mkdirSync, writeFileSync } from "node:fs"; import { relative, resolve as resolvePath } from "node:path"; import type { ApprovalRequest } from "./approval-types.js"; import { redactSecrets } from "./redaction.js"; +import { trimBoundaryCharacters } from "./string-boundaries.js"; import { DEFAULT_WORKSPACE_ID } from "./workspace-types.js"; import { normalizeGitHubExternalWorkItem } from "./external-work.js"; import type { @@ -145,7 +146,7 @@ export function createGitHubExternalActionAdapter(options: GitHubExternalActionA if (!fetchImpl) { throw new Error("A fetch implementation is required for GitHub external action execution."); } - const apiBaseUrl = (options.apiBaseUrl ?? "https://api.github.com").replace(/\/+$/, ""); + const apiBaseUrl = trimBoundaryCharacters(options.apiBaseUrl ?? "https://api.github.com", "/", "end"); return { async getPullRequest(input: GitHubSubjectInput): Promise { diff --git a/packages/core/src/external-work-ci-triage.ts b/packages/core/src/external-work-ci-triage.ts index 9b0c8e3e3..4b6587e50 100644 --- a/packages/core/src/external-work-ci-triage.ts +++ b/packages/core/src/external-work-ci-triage.ts @@ -3,6 +3,7 @@ import { mkdirSync, writeFileSync } from "node:fs"; import { relative, resolve as resolvePath } from "node:path"; import { normalizeGitHubExternalWorkItem } from "./external-work.js"; import { redactSecrets } from "./redaction.js"; +import { trimBoundaryCharacters } from "./string-boundaries.js"; import type { ExternalWorkCiTriageRecord, ExternalWorkCiTriageResponse, @@ -85,7 +86,7 @@ export function createGitHubCiTriageAdapter(options: GitHubCiTriageAdapterOption if (!fetchImpl) { throw new Error("A fetch implementation is required for GitHub CI triage."); } - const apiBaseUrl = (options.apiBaseUrl ?? "https://api.github.com").replace(/\/+$/, ""); + const apiBaseUrl = trimBoundaryCharacters(options.apiBaseUrl ?? "https://api.github.com", "/", "end"); return { async getPullRequest(input): Promise { diff --git a/packages/core/src/external-work-scheduler.ts b/packages/core/src/external-work-scheduler.ts index ab9e01213..b772a96ea 100644 --- a/packages/core/src/external-work-scheduler.ts +++ b/packages/core/src/external-work-scheduler.ts @@ -4,6 +4,7 @@ import { existsSync, mkdirSync, writeFileSync } from "node:fs"; import { isAbsolute, relative, resolve as resolvePath } from "node:path"; import { DEFAULT_WORKSPACE_ID, type Channel } from "./workspace-types.js"; import { redactSecrets } from "./redaction.js"; +import { trimBoundaryCharacters } from "./string-boundaries.js"; import { createCoreOriginalProcessEntry, isCoreOriginalLaunchRefusal, type CoreOriginalProcessEntry } from "./original-host-process-entry.js"; import type { ChannelStoreFacet, @@ -117,7 +118,7 @@ export function createGitHubExternalWorkAdapter(options: GitHubExternalWorkAdapt if (!fetchImpl) { throw new Error("A fetch implementation is required for GitHub external work polling."); } - const apiBaseUrl = (options.apiBaseUrl ?? "https://api.github.com").replace(/\/+$/, ""); + const apiBaseUrl = trimBoundaryCharacters(options.apiBaseUrl ?? "https://api.github.com", "/", "end"); return { id: "github-readonly", diff --git a/packages/core/src/external-work.ts b/packages/core/src/external-work.ts index 861748f36..b5c20fc9f 100644 --- a/packages/core/src/external-work.ts +++ b/packages/core/src/external-work.ts @@ -59,17 +59,45 @@ export function updateExternalWorkflowSecretRef(markdown: string, secretRef: str } export function renderExternalWorkflowPrompt(template: string, input: { readonly item: ExternalWorkItem }): string { - return template.replace(/\{\{\s*([^}]+?)\s*\}\}/g, (_match, expression: string) => { - const path = expression.trim(); - if (!path.startsWith("item.")) { - throw new ExternalWorkflowValidationError(`Unknown workflow prompt variable: ${path}`); + const parts: string[] = []; + let cursor = 0; + let opening: number | undefined; + let index = 0; + while (index < template.length) { + const character = template[index]; + const nextCharacter = template[index + 1]; + if (character === "{" && nextCharacter === "{") { + if (opening === undefined) opening = index; + index += 1; + continue; } - const value = readDotPath(input, path); - if (value === undefined || value === null) { - throw new ExternalWorkflowValidationError(`Unknown workflow prompt variable: ${path}`); + if (character !== "}") { + index += 1; + continue; } - return formatPromptValue(value); - }); + if (nextCharacter !== "}") { + opening = undefined; + index += 1; + continue; + } + if (opening !== undefined && index > opening + 2) { + parts.push(template.slice(cursor, opening)); + const path = template.slice(opening + 2, index).trim(); + if (!path.startsWith("item.")) { + throw new ExternalWorkflowValidationError(`Unknown workflow prompt variable: ${path}`); + } + const value = readDotPath(input, path); + if (value === undefined || value === null) { + throw new ExternalWorkflowValidationError(`Unknown workflow prompt variable: ${path}`); + } + parts.push(formatPromptValue(value)); + cursor = index + 2; + } + opening = undefined; + index += 2; + } + parts.push(template.slice(cursor)); + return parts.join(""); } export function normalizeGitHubExternalWorkItem(input: { diff --git a/packages/core/src/managed-official-skill-source.ts b/packages/core/src/managed-official-skill-source.ts index 6ccdd37cb..e268e8112 100644 --- a/packages/core/src/managed-official-skill-source.ts +++ b/packages/core/src/managed-official-skill-source.ts @@ -4,6 +4,7 @@ import { join, resolve } from "node:path"; import { promisify } from "node:util"; import type { SkillSourceScanRoot } from "./skill-types.js"; import { createCoreOriginalProcessEntry, isCoreOriginalLaunchRefusal } from "./original-host-process-entry.js"; +import { trimBoundaryCharacters } from "./string-boundaries.js"; const execFile = promisify(execFileCallback); @@ -179,7 +180,7 @@ async function readGstackVersion(repoRoot: string): Promise function normalizeGithubPath(host: string, pathname: string): string | undefined { const normalizedHost = host.trim().toLowerCase(); - const normalizedPath = pathname.replace(/^\/+|\/+$/g, "").replace(/\.git$/i, "").toLowerCase(); + const normalizedPath = trimBoundaryCharacters(pathname, "/").replace(/\.git$/i, "").toLowerCase(); return normalizedHost && normalizedPath ? `${normalizedHost}/${normalizedPath}` : undefined; } diff --git a/packages/core/src/memory-markdown-import.ts b/packages/core/src/memory-markdown-import.ts index b2ecf8dae..6d011ca8e 100644 --- a/packages/core/src/memory-markdown-import.ts +++ b/packages/core/src/memory-markdown-import.ts @@ -5,6 +5,7 @@ import type { MemorySourceRef, } from "./memory-types.js"; import { DEFAULT_WORKSPACE_ID } from "./workspace-types.js"; +import { trimBoundaryCharacters } from "./string-boundaries.js"; export interface MemoryMarkdownImportOptions { readonly workspaceId?: string; @@ -374,7 +375,7 @@ function importSourceRefId(sourceId: string, itemId: string): string { } function sanitizeSourceId(value: string): string { - const sanitized = value.trim().replace(/[^A-Za-z0-9._:-]+/g, "_").replace(/^_+|_+$/g, ""); + const sanitized = trimBoundaryCharacters(value.trim().replace(/[^A-Za-z0-9._:-]+/g, "_"), "_"); return sanitized || "memory-markdown-import"; } diff --git a/packages/core/src/provider-adapters.ts b/packages/core/src/provider-adapters.ts index 1e196e162..32d2a36b7 100644 --- a/packages/core/src/provider-adapters.ts +++ b/packages/core/src/provider-adapters.ts @@ -4,6 +4,7 @@ import { isKnownGenerationModel, isOpenAIImagesGenerationModel, } from "./provider-generation-models.js"; +import { trimBoundaryCharacters } from "./string-boundaries.js"; export type ProviderAdapterMediaKind = "seedream" | "seedance" | "jimeng"; @@ -97,10 +98,10 @@ export function builtinMediaModelCandidates(kind: ProviderAdapterMediaKind | und export function normalizeProviderBaseUrlForAdapter(input: Pick): string { const rawBaseUrl = input.baseUrl?.trim() || defaultProviderBaseUrl(input); - const withoutTrailingSlash = rawBaseUrl.replace(/\/+$/, ""); + const withoutTrailingSlash = trimBoundaryCharacters(rawBaseUrl, "/", "end"); try { const url = new URL(withoutTrailingSlash); - let pathname = url.pathname.replace(/\/+$/, ""); + let pathname = trimBoundaryCharacters(url.pathname, "/", "end"); for (const suffix of endpointSuffixes) { if (pathname.endsWith(suffix)) { pathname = pathname.slice(0, -suffix.length); @@ -110,7 +111,7 @@ export function normalizeProviderBaseUrlForAdapter(input: Pick key(candidate) === key(root) || key(candidate).startsWith(key(root) + paths.sep); diff --git a/packages/core/src/skill-optimization-preview.ts b/packages/core/src/skill-optimization-preview.ts index 0e3925f2f..054a49efc 100644 --- a/packages/core/src/skill-optimization-preview.ts +++ b/packages/core/src/skill-optimization-preview.ts @@ -1,5 +1,6 @@ import { createHash } from "node:crypto"; import { buildSkillSourceContextFindings } from "./skill-source-health.js"; +import { parseSkillFrontMatter } from "./skill-source-definition.js"; import type { SkillCompiledRuntimeView, SkillCompiledSourceMapEntry, @@ -264,20 +265,11 @@ function findRoutingDescription(snapshot: SkillSourceSnapshot): string | undefin } function parseFrontMatterName(markdown: string): string | undefined { - return parseFrontMatterField(markdown, "name"); + return parseSkillFrontMatter(markdown).frontMatter.name; } function parseFrontMatterDescription(markdown: string): string | undefined { - return parseFrontMatterField(markdown, "description"); -} - -function parseFrontMatterField(markdown: string, field: string): string | undefined { - const frontMatter = /^---\n([\s\S]*?)\n---/u.exec(markdown)?.[1]; - if (!frontMatter) { - return undefined; - } - const match = new RegExp(`^${field}:\\s*["']?(.+?)["']?\\s*$`, "imu").exec(frontMatter)?.[1]; - return match && match !== "|" && match !== ">" ? match : undefined; + return parseSkillFrontMatter(markdown).frontMatter.description; } function estimateOriginalTokens(snapshot: SkillSourceSnapshot): number { diff --git a/packages/core/src/skill-source-health.ts b/packages/core/src/skill-source-health.ts index 685f75f32..3969ab013 100644 --- a/packages/core/src/skill-source-health.ts +++ b/packages/core/src/skill-source-health.ts @@ -1,4 +1,5 @@ import { GSTACK_DEFAULT_SKILL_PACK_ID } from "./builtin-skill-packs.js"; +import { parseSkillFrontMatter } from "./skill-source-definition.js"; import type { SkillRegistryEntry, SkillSourceContextFindingEvidence, @@ -341,19 +342,7 @@ function findRoutingDescription(snapshot: SkillSourceSnapshot): string | undefin } function parseFrontMatterDescription(markdown: string): string | undefined { - const frontMatter = /^---\n([\s\S]*?)\n---/u.exec(markdown)?.[1]; - if (!frontMatter) { - return undefined; - } - const singleLine = /^description:\s*["']?(.+?)["']?\s*$/imu.exec(frontMatter)?.[1]; - if (singleLine && singleLine !== "|" && singleLine !== ">") { - return singleLine; - } - const block = /^description:\s*[|>]\s*\n((?:[ \t]+.+\n?)+)/imu.exec(frontMatter)?.[1]; - if (block) { - return block.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean).join(" "); - } - return undefined; + return parseSkillFrontMatter(markdown).frontMatter.description; } function stripFrontMatter(markdown: string): string { diff --git a/packages/core/src/skill-source-sync.ts b/packages/core/src/skill-source-sync.ts index 2c3c5ac08..1557b1374 100644 --- a/packages/core/src/skill-source-sync.ts +++ b/packages/core/src/skill-source-sync.ts @@ -6,6 +6,7 @@ import { parseDocument } from "yaml"; import { buildSkillSourceContextFindings } from "./skill-source-health.js"; import { parseSkillFrontMatter, previewSkillSourceDefinition, splitMarkdownFrontMatter } from "./skill-source-definition.js"; import { selectActiveSkillSourceSnapshots } from "./skill-source-selection.js"; +import { collapseRepeatedCharacter, trimBoundaryCharacters } from "./string-boundaries.js"; import type { CreateSkillSourceSnapshotInput, SkillAgentMetadata, @@ -293,14 +294,10 @@ export async function previewGstackSkillSourceSync(input: PreviewGstackSkillSour } export function normalizeGstackSkillId(input: string): string { - return input - .trim() - .toLowerCase() - .replace(/^\/+|\/+$/g, "") + const normalized = trimBoundaryCharacters(input.trim().toLowerCase(), "/") .replace(/\s+/g, "-") - .replace(/[^a-z0-9._-]+/g, "-") - .replace(/-+/g, "-") - .replace(/^-|-$/g, ""); + .replace(/[^a-z0-9._-]+/g, "-"); + return trimBoundaryCharacters(collapseRepeatedCharacter(normalized, "-"), "-"); } async function gstackRootFromRepoPath(repoPath: string, id?: string): Promise { @@ -513,10 +510,15 @@ function classifySkillSourceWarnings(markdown: string, sourceSizeBytes: number, /\$GSTACK_BIN\b/, /\bgstack-/, ], "SKILL.md references host-specific gstack or Claude runtime behavior."); - addPatternWarning(warnings, markdown, "unsupported_tool", "warning", [ - /\bmcp__[A-Za-z0-9_-]+__[A-Za-z0-9_-]+\b/, - /\bnative AUQ\b/i, - ], "SKILL.md references a tool surface that Ling may not expose directly."); + const unsupportedToolLine = findFirstMcpToolLine(markdown) ?? findFirstMatchingLine(markdown, /\bnative AUQ\b/i); + if (unsupportedToolLine !== undefined) { + warnings.push({ + code: "unsupported_tool", + severity: "warning", + message: "SKILL.md references a tool surface that Ling may not expose directly.", + location: `SKILL.md:${unsupportedToolLine}`, + }); + } addPatternWarning(warnings, markdown, "permission_like_instruction", "info", [ /\b(approval bypass|bypass approval|ignore approval|grant .*tool|allowedTools|disallowedTools|permission gate)\b/i, ], "SKILL.md contains permission-like language that must not override Ling policy."); @@ -609,7 +611,10 @@ export function buildSkillSourceSemanticProfile(input: { workflowVerbs: uniqueSortedClaims(extractClaims(input.markdown, workflowVerbPatterns)), stopGateCount: countPattern(input.markdown, /\b(STOP|approval gate|ask for approval|ask the user|confirm before|requires confirmation)\b/gi), approvalGateClaims: uniqueSortedClaims(extractClaims(input.markdown, approvalGatePatterns)), - toolClaims: uniqueSortedClaims(extractClaims(input.markdown, toolClaimPatterns)), + toolClaims: uniqueSortedClaims([ + ...extractClaims(input.markdown, toolClaimPatterns), + ...extractMcpToolClaims(input.markdown), + ]), hostClaims: uniqueSortedClaims(extractClaims(input.markdown, hostClaimPatterns)), externalActionClaims: uniqueSortedClaims(extractClaims(input.markdown, externalActionPatterns)), destructiveClaims: uniqueSortedClaims(extractClaims(input.markdown, destructiveClaimPatterns)), @@ -719,7 +724,7 @@ const approvalGatePatterns = [ ] as const; const toolClaimPatterns = [ - /\b(bash|shell|filesystem|computer-use|browser|playwright|apply_patch|mcp__[A-Za-z0-9_-]+__[A-Za-z0-9_-]+|git|gh)\b/gi, + /\b(bash|shell|filesystem|computer-use|browser|playwright|apply_patch|git|gh)\b/gi, ] as const; const hostClaimPatterns = [ @@ -767,6 +772,46 @@ function extractClaims(markdown: string, patterns: readonly RegExp[]): string[] return claims; } +function extractMcpToolClaims(markdown: string): string[] { + const claims: string[] = []; + const lowerMarkdown = markdown.toLowerCase(); + for (let index = 0; index < markdown.length;) { + const start = lowerMarkdown.indexOf("mcp__", index); + if (start === -1) break; + if (isMcpToolCharacter(markdown[start - 1])) { + index = start + 1; + continue; + } + let end = start + "mcp__".length; + while (end < markdown.length && isMcpToolCharacter(markdown[end])) end += 1; + const candidate = markdown.slice(start, end); + const components = candidate.slice("mcp__".length); + const separator = components.indexOf("__", 1); + if (separator !== -1 && separator + 2 < components.length) claims.push(candidate.toLowerCase()); + index = Math.max(end, start + 1); + } + return claims; +} + +function findFirstMcpToolLine(markdown: string): number | undefined { + const claim = extractMcpToolClaims(markdown)[0]; + if (!claim) return undefined; + const index = markdown.toLowerCase().indexOf(claim); + if (index === -1) return undefined; + let line = 1; + for (let cursor = 0; cursor < index; cursor += 1) { + if (markdown[cursor] === "\n") line += 1; + } + return line; +} + +function isMcpToolCharacter(value: string | undefined): boolean { + if (!value) return false; + const code = value.charCodeAt(0); + return (code >= 48 && code <= 57) || (code >= 65 && code <= 90) || + (code >= 97 && code <= 122) || value === "_" || value === "-"; +} + function uniqueSortedClaims(values: readonly string[]): string[] { return [...new Set(values.filter(Boolean))].sort(); } diff --git a/packages/core/src/string-boundaries.ts b/packages/core/src/string-boundaries.ts new file mode 100644 index 000000000..84e89dc43 --- /dev/null +++ b/packages/core/src/string-boundaries.ts @@ -0,0 +1,93 @@ +export function trimBoundaryCharacters( + value: string, + characters: string, + side: "start" | "end" | "both" = "both", +): string { + let start = 0; + let end = value.length; + if (side !== "end") { + while (start < end && characters.includes(value[start] ?? "")) start += 1; + } + if (side !== "start") { + while (end > start && characters.includes(value[end - 1] ?? "")) end -= 1; + } + return start === 0 && end === value.length ? value : value.slice(start, end); +} + +export function collapseRepeatedCharacter(value: string, character: string): string { + const result: string[] = []; + let previousWasCharacter = false; + for (const current of value) { + if (current === character) { + if (!previousWasCharacter) result.push(current); + previousWasCharacter = true; + } else { + result.push(current); + previousWasCharacter = false; + } + } + return result.join(""); +} + +export function extractWrappedSection( + value: string, + labels: readonly string[], + boundaryLabels: readonly string[], + acceptedColons = "::", + allowWhitespaceBeforeColon = true, +): string | undefined { + const searchValue = asciiLowercase(value); + let selected: { readonly labelIndex: number; readonly start: number } | undefined; + for (const label of labels) { + const searchLabel = asciiLowercase(label); + let labelIndex = searchValue.indexOf(searchLabel); + while (labelIndex !== -1) { + let start = labelIndex + label.length; + if (allowWhitespaceBeforeColon) { + while (start < value.length && isWhitespace(value[start])) start += 1; + } + if (acceptedColons.includes(value[start] ?? "")) { + start += 1; + while (start < value.length && isWhitespace(value[start])) start += 1; + if (!selected || labelIndex < selected.labelIndex) selected = { labelIndex, start }; + break; + } + labelIndex = searchValue.indexOf(searchLabel, labelIndex + 1); + } + } + if (!selected) return undefined; + let end = value.length; + for (const boundary of boundaryLabels) { + const searchBoundary = asciiLowercase(boundary); + let boundaryIndex = searchValue.indexOf(searchBoundary, selected.start); + while (boundaryIndex !== -1) { + const before = value[boundaryIndex - 1]; + const after = value[boundaryIndex + boundary.length]; + if (isWhitespace(before) && !isWordCharacter(after)) { + end = Math.min(end, boundaryIndex); + break; + } + boundaryIndex = searchValue.indexOf(searchBoundary, boundaryIndex + 1); + } + } + const section = value.slice(selected.start, end).trim(); + return section || undefined; +} + +function asciiLowercase(value: string): string { + const result: string[] = []; + for (let index = 0; index < value.length; index += 1) { + const character = value[index] ?? ""; + const code = character.charCodeAt(0); + result.push(code >= 65 && code <= 90 ? String.fromCharCode(code + 32) : character); + } + return result.join(""); +} + +function isWhitespace(value: string | undefined): boolean { + return value !== undefined && value.trim() === ""; +} + +function isWordCharacter(value: string | undefined): boolean { + return value !== undefined && /[A-Za-z0-9_]/u.test(value); +} diff --git a/packages/core/src/terminal-report-fallback.ts b/packages/core/src/terminal-report-fallback.ts index 5a9509670..6f1909de8 100644 --- a/packages/core/src/terminal-report-fallback.ts +++ b/packages/core/src/terminal-report-fallback.ts @@ -11,6 +11,7 @@ import type { } from "./terminal-report-types.js"; import { createTerminalReportId } from "./terminal-report-types.js"; import { formatSynchronousCatalogMessage } from "@ling-stack/localization"; +import { extractWrappedSection, trimBoundaryCharacters } from "./string-boundaries.js"; export interface BuildDeterministicTerminalReportInput { readonly id?: string; @@ -113,10 +114,14 @@ export function buildDeterministicTerminalReportContent(snapshot: TerminalReport */ function fallbackOutcomeTitle(snapshot: TerminalReportEvidenceSnapshot): string { const normalized = snapshot.objective.replace(/\s+/gu, " ").trim(); - const wrapped = normalized.match( - /(?:原始请求|Original request):\s*(.+?)(?=\s+(?:Requested objective\b|No literal file content\b|已物化的执行操作\b|参与者操作归属\b|Permission boundary\b)|$)/iu, - )?.[1]?.trim(); - const candidate = (wrapped || normalized).replace(/^\*+|\*+$/gu, "").trim(); + const wrapped = extractWrappedSection(normalized, ["原始请求", "Original request"], [ + "Requested objective", + "No literal file content", + "已物化的执行操作", + "参与者操作归属", + "Permission boundary", + ], ":", false); + const candidate = trimBoundaryCharacters(wrapped || normalized, "*").trim(); const maxLength = 120; return candidate.length > maxLength ? `${candidate.slice(0, maxLength - 1).trimEnd()}…` : candidate; } diff --git a/packages/core/test/external-work.test.mjs b/packages/core/test/external-work.test.mjs index e80f30350..fb32a9424 100644 --- a/packages/core/test/external-work.test.mjs +++ b/packages/core/test/external-work.test.mjs @@ -45,6 +45,11 @@ Labels: {{ item.labels }} Head: {{ item.headSha }} `; +test("workflow prompt rendering handles long malformed placeholders without backtracking", () => { + const template = `${"{{".repeat(50_000)}${" ".repeat(100_000)}`; + assert.equal(renderExternalWorkflowPrompt(template, { item: { identifier: "#1" } }), template); +}); + const linearWorkflowMarkdown = `--- source: kind: linear diff --git a/packages/core/test/provider-check.test.mjs b/packages/core/test/provider-check.test.mjs index 58d8a2652..2f4a40c59 100644 --- a/packages/core/test/provider-check.test.mjs +++ b/packages/core/test/provider-check.test.mjs @@ -33,6 +33,11 @@ test("default provider templates include Seedream live HTTP transport config", ( assert.equal(seedream.enabled, false); }); +test("BFL edit capability inference handles long model identifiers", () => { + const model = `black-forest-labs/flux-${"flux-".repeat(20_000)}kontext-pro`; + assert.deepEqual(inferGenerationModelFamily(model)?.capabilities, ["image_generate", "image_edit"]); +}); + test("official provider templates expose fixed default base URLs", () => { const templates = defaultProviderTemplates(); const byId = new Map(templates.map((template) => [template.templateId, template])); diff --git a/packages/core/test/skill-source-sync.test.mjs b/packages/core/test/skill-source-sync.test.mjs index f917dd720..66a451bbb 100644 --- a/packages/core/test/skill-source-sync.test.mjs +++ b/packages/core/test/skill-source-sync.test.mjs @@ -11,6 +11,7 @@ import { getBuiltinSkillPack, ManagedOfficialSkillSourceError, normalizeOfficialGstackOrigin, + normalizeGstackSkillId, officialGstackSourceScanRoot, previewGstackSkillSourceSync, refreshManagedOfficialGstackSource, @@ -18,6 +19,13 @@ import { resolveSkillSourceRoots, } from "../dist/index.js"; +test("skill identifiers and MCP claims handle long ambiguous delimiters", () => { + assert.equal(normalizeGstackSkillId(`${"/".repeat(100_000)}review${"-".repeat(100_000)}`), "review"); + const tool = `mcp__server${"_".repeat(100_000)}__tool`; + const profile = buildSkillSourceSemanticProfile({ markdown: `Use ${tool} and MCP__Server__Tool, but not mcp____.`, sourceChecksum: "sha256:long" }); + assert.deepEqual(profile.toolClaims, [tool, "mcp__server__tool"]); +}); + async function withTempDir(prefix, fn) { const root = await mkdtemp(resolve(tmpdir(), prefix)); try { diff --git a/packages/daemon/src/features/channels/link-previews.ts b/packages/daemon/src/features/channels/link-previews.ts index eb2ca5a6d..19ccb4b18 100644 --- a/packages/daemon/src/features/channels/link-previews.ts +++ b/packages/daemon/src/features/channels/link-previews.ts @@ -7,6 +7,7 @@ import { writeJson } from "../../shared/response.js"; import { preparePinnedPublicHttpRequest, PublicHttpError } from "../../shared/public-http-transport.js"; import { validateChannelMessageOwner } from "./protocol.js"; import { parseChannelLinkPreviewRequest, type ChannelLinkPreviewRequest } from "./schemas.js"; +import { decodeHtmlEntitiesOnce } from "../../shared/html-text.js"; const CHANNEL_LINK_PREVIEW_TIMEOUT_MS = 5_000; const CHANNEL_LINK_PREVIEW_MAX_BYTES = 192 * 1024; @@ -251,13 +252,7 @@ function readTitle(html: string): string | undefined { } function decodeHtmlEntities(value: string): string { - return value - .replace(/&/g, "&") - .replace(/</g, "<") - .replace(/>/g, ">") - .replace(/"/g, "\"") - .replace(/'/g, "'") - .trim(); + return decodeHtmlEntitiesOnce(value, { names: ["amp", "lt", "gt", "quot", "#39"] }).trim(); } export function normalizePreviewUrl(rawUrl: string): string { diff --git a/packages/daemon/src/features/channels/materialization/extractors/link-body.ts b/packages/daemon/src/features/channels/materialization/extractors/link-body.ts index 6e82e9d35..4421457e7 100644 --- a/packages/daemon/src/features/channels/materialization/extractors/link-body.ts +++ b/packages/daemon/src/features/channels/materialization/extractors/link-body.ts @@ -1,6 +1,7 @@ import type { LingInputExtractor } from "../registry.js"; import { decidePublicLinkMaterializationPolicy } from "../policy.js"; import { preparePinnedPublicHttpRequest, PublicHttpError } from "../../../../shared/public-http-transport.js"; +import { decodeHtmlEntitiesOnce, stripUnreadableHtml } from "../../../../shared/html-text.js"; export const linkBodyExtractorV1: LingInputExtractor = { id: "link_body_v1", @@ -242,18 +243,7 @@ function isSupportedLinkBodyContentType(contentType: string): boolean { } export function extractReadableHtmlText(html: string): string { - const body = /]*>([\s\S]*?)<\/body>/iu.exec(html)?.[1] ?? html; - return normalizeExtractedText( - body - .replace(//giu, " ") - .replace(//giu, " ") - .replace(//giu, " ") - .replace(//giu, " ") - .replace(/<(?:nav|footer|form|aside)\b[\s\S]*?<\/(?:nav|footer|form|aside)>/giu, " ") - .replace(//giu, "\n") - .replace(/<\/(?:p|div|section|article|main|h[1-6]|li|tr)>/giu, "\n") - .replace(/<[^>]+>/gu, " "), - ); + return normalizeExtractedText(stripUnreadableHtml(html)); } export function normalizeExtractedText(value: string): string { @@ -266,15 +256,7 @@ export function normalizeExtractedText(value: string): string { } function decodeHtmlEntities(value: string): string { - return value - .replace(/ /g, " ") - .replace(/&/g, "&") - .replace(/</g, "<") - .replace(/>/g, ">") - .replace(/"/g, "\"") - .replace(/'/g, "'") - .replace(/&#(\d+);/g, (_match, code: string) => String.fromCodePoint(Number.parseInt(code, 10))) - .replace(/&#x([0-9a-f]+);/giu, (_match, code: string) => String.fromCodePoint(Number.parseInt(code, 16))); + return decodeHtmlEntitiesOnce(value, { numeric: true }); } function publicLinkPolicyMessage(reason: string): string { diff --git a/packages/daemon/src/features/web/web-search-adapters.ts b/packages/daemon/src/features/web/web-search-adapters.ts index 370fee56e..047946b17 100644 --- a/packages/daemon/src/features/web/web-search-adapters.ts +++ b/packages/daemon/src/features/web/web-search-adapters.ts @@ -2,6 +2,7 @@ import { normalizeProviderBaseUrlForAdapter, type ProviderPreset, } from "@ling-stack/core"; +import { decodeHtmlEntitiesOnce } from "../../shared/html-text.js"; const DEFAULT_OPENAI_WEB_SEARCH_MODEL = "gpt-5.5"; const WEB_SEARCH_MAX_OUTPUT_TOKENS = 900; @@ -685,15 +686,13 @@ function isPublicHttpUrl(value: string): boolean { } function cleanMarkdownText(value: string): string { - return value + const cleaned = value .replace(/\*\*/gu, "") .replace(/!\[[^\]]*\]\([^)]+\)/gu, "") .replace(/\[[^\]]*\]\([^)]+\)/gu, "") - .replace(/&/gu, "&") - .replace(/"/gu, "\"") - .replace(/'/gu, "'") .replace(/\s+/gu, " ") .trim(); + return decodeHtmlEntitiesOnce(cleaned, { names: ["amp", "quot", "#39"] }); } function looksLikeBareHost(value: string): boolean { diff --git a/packages/daemon/src/shared/html-text.ts b/packages/daemon/src/shared/html-text.ts new file mode 100644 index 000000000..6b7ed87e1 --- /dev/null +++ b/packages/daemon/src/shared/html-text.ts @@ -0,0 +1,153 @@ +interface HtmlEntityDecodeOptions { + readonly names?: readonly ("nbsp" | "amp" | "lt" | "gt" | "quot" | "#39")[]; + readonly numeric?: boolean; +} + +const htmlEntityPattern = /&(?:nbsp|amp|lt|gt|quot|#39|#\d{1,7}|#x[0-9a-f]{1,6});/giu; + +export function decodeHtmlEntitiesOnce(value: string, options: HtmlEntityDecodeOptions = {}): string { + const names = new Set(options.names ?? ["nbsp", "amp", "lt", "gt", "quot", "#39"]); + return value.replace(htmlEntityPattern, (entity) => { + const token = entity.slice(1, -1).toLowerCase(); + if (token.startsWith("#") && token !== "#39") { + if (options.numeric !== true) return entity; + const radix = token.startsWith("#x") ? 16 : 10; + const digits = token.slice(radix === 16 ? 2 : 1); + const codePoint = Number.parseInt(digits, radix); + return isValidCodePoint(codePoint) ? String.fromCodePoint(codePoint) : entity; + } + if (!names.has(token as "nbsp" | "amp" | "lt" | "gt" | "quot" | "#39")) return entity; + switch (token) { + case "nbsp": return " "; + case "amp": return "&"; + case "lt": return "<"; + case "gt": return ">"; + case "quot": return "\""; + case "#39": return "'"; + default: return entity; + } + }); +} + +export function stripUnreadableHtml(html: string): string { + const body = findBodyContentRange(html); + const source = body ? html.slice(body.start, body.end) : html; + const output: string[] = []; + let cursor = 0; + while (cursor < source.length) { + if (source[cursor] !== "<") { + output.push(source[cursor] ?? ""); + cursor += 1; + continue; + } + if (source.startsWith("", cursor + 4); + cursor = commentEnd === -1 ? source.length : commentEnd + 3; + output.push(" "); + continue; + } + const tag = readHtmlTag(source, cursor); + if (!tag) { + output.push("<"); + cursor += 1; + continue; + } + cursor = tag.end; + if (!tag.closing && unreadableTags.has(tag.name)) { + cursor = findClosingTag(source, tag.name, cursor)?.end ?? source.length; + output.push(" "); + continue; + } + if (tag.name === "br" || (tag.closing && blockTags.has(tag.name))) output.push("\n"); + else output.push(" "); + } + return output.join(""); +} + +const unreadableTags = new Set(["script", "style", "noscript", "svg", "nav", "footer", "form", "aside", "head"]); +const blockTags = new Set(["p", "div", "section", "article", "main", "h1", "h2", "h3", "h4", "h5", "h6", "li", "tr"]); + +function findBodyContentRange(html: string): { readonly start: number; readonly end: number } | undefined { + let cursor = 0; + while (cursor < html.length) { + const opening = html.indexOf("<", cursor); + if (opening === -1) return undefined; + const tag = readHtmlTag(html, opening); + if (!tag) { + cursor = opening + 1; + continue; + } + if (!tag.closing && tag.name === "body") { + const closing = findClosingTag(html, "body", tag.end); + return { start: tag.end, end: closing?.start ?? html.length }; + } + if (!tag.closing && unreadableTags.has(tag.name)) { + cursor = findClosingTag(html, tag.name, tag.end)?.end ?? html.length; + } else { + cursor = tag.end; + } + } + return undefined; +} + +function findClosingTag(html: string, name: string, start: number): HtmlTag | undefined { + let cursor = start; + while (cursor < html.length) { + const opening = html.indexOf("<", cursor); + if (opening === -1) return undefined; + const tag = readHtmlTag(html, opening); + if (tag?.closing && tag.name === name) return tag; + cursor = tag ? tag.end : opening + 1; + } + return undefined; +} + +interface HtmlTag { + readonly start: number; + readonly name: string; + readonly closing: boolean; + readonly end: number; +} + +function readHtmlTag(html: string, start: number): HtmlTag | undefined { + if (html[start] !== "<") return undefined; + let cursor = start + 1; + while (isHtmlWhitespace(html[cursor])) cursor += 1; + let closing = false; + if (html[cursor] === "/") { + closing = true; + cursor += 1; + while (isHtmlWhitespace(html[cursor])) cursor += 1; + } + const nameStart = cursor; + while (isAsciiLetterOrDigit(html[cursor])) cursor += 1; + if (cursor === nameStart) return undefined; + const name = html.slice(nameStart, cursor).toLowerCase(); + let quote: "\"" | "'" | undefined; + while (cursor < html.length) { + const character = html[cursor]; + if (quote) { + if (character === quote) quote = undefined; + } else if (character === "\"" || character === "'") { + quote = character; + } else if (character === ">") { + return { start, name, closing, end: cursor + 1 }; + } + cursor += 1; + } + return undefined; +} + +function isHtmlWhitespace(value: string | undefined): boolean { + return value === " " || value === "\t" || value === "\n" || value === "\r" || value === "\f"; +} + +function isAsciiLetterOrDigit(value: string | undefined): boolean { + if (!value) return false; + const code = value.charCodeAt(0); + return (code >= 48 && code <= 57) || (code >= 65 && code <= 90) || (code >= 97 && code <= 122); +} + +function isValidCodePoint(value: number): boolean { + return Number.isInteger(value) && value >= 0 && value <= 0x10ffff && !(value >= 0xd800 && value <= 0xdfff); +} diff --git a/packages/daemon/test/html-text-security.test.mjs b/packages/daemon/test/html-text-security.test.mjs new file mode 100644 index 000000000..d1b75565d --- /dev/null +++ b/packages/daemon/test/html-text-security.test.mjs @@ -0,0 +1,14 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { extractReadableHtmlText } from "../dist/features/channels/materialization/extractors/link-body.js"; +import { decodeHtmlEntitiesOnce } from "../dist/shared/html-text.js"; + +test("readable HTML extraction removes whitespace-terminated script tags", () => { + const text = extractReadableHtmlText("ignored before

before

after

ignored after"); + assert.equal(text, "before\nafter"); +}); + +test("HTML entities decode exactly one layer", () => { + assert.equal(decodeHtmlEntitiesOnce("&lt; < &quot;"), "< < ""); + assert.equal(decodeHtmlEntitiesOnce("< <", { numeric: true }), "< <"); +}); diff --git a/packages/providers/src/utils.ts b/packages/providers/src/utils.ts index b2dac747c..113041975 100644 --- a/packages/providers/src/utils.ts +++ b/packages/providers/src/utils.ts @@ -5,3 +5,13 @@ export function isRecord(value: unknown): value is Record { export function removeUndefined>(value: T): T { return Object.fromEntries(Object.entries(value).filter(([, item]) => item !== undefined)) as T; } + +export function stripTrailingSlashes(value: string): string; +export function stripTrailingSlashes(value: undefined): undefined; +export function stripTrailingSlashes(value: string | undefined): string | undefined; +export function stripTrailingSlashes(value: string | undefined): string | undefined { + if (value === undefined) return undefined; + let end = value.length; + while (end > 0 && value[end - 1] === "/") end -= 1; + return end === value.length ? value : value.slice(0, end); +} diff --git a/packages/providers/src/vendors/openai-images.ts b/packages/providers/src/vendors/openai-images.ts index 9b5a6d026..39515e426 100644 --- a/packages/providers/src/vendors/openai-images.ts +++ b/packages/providers/src/vendors/openai-images.ts @@ -10,7 +10,7 @@ import type { MediaVendorTransportRequest, MediaVendorTransportResult, } from "../types.js"; -import { isRecord, removeUndefined } from "../utils.js"; +import { isRecord, removeUndefined, stripTrailingSlashes } from "../utils.js"; const MAX_OPENAI_IMAGE_BYTES = 25 * 1024 * 1024; const DEFAULT_OPENAI_IMAGES_TIMEOUT_MS = 180_000; @@ -19,7 +19,7 @@ export async function executeOpenAIImageGeneration( request: MediaVendorTransportRequest, options: LiveMediaVendorTransportOptions, ): Promise { - const endpoint = `${options.baseUrl?.replace(/\/+$/, "")}/images/generations`; + const endpoint = `${stripTrailingSlashes(options.baseUrl)}/images/generations`; const payload = buildOpenAIImageGenerationPayload(request.requestPlan); const httpClient = options.httpClient ?? defaultMediaVendorHttpClient; const abortController = new AbortController(); diff --git a/packages/providers/src/vendors/seedance-task-status.ts b/packages/providers/src/vendors/seedance-task-status.ts index 5632f3aef..eac624503 100644 --- a/packages/providers/src/vendors/seedance-task-status.ts +++ b/packages/providers/src/vendors/seedance-task-status.ts @@ -1,4 +1,4 @@ -import { isRecord } from "../utils.js"; +import { isRecord, stripTrailingSlashes } from "../utils.js"; import type { MediaRemoteTerminalObservation } from "@ling-stack/contracts"; /** Decoded protocol facts only. Host transport and Core lineage still decide settlement. */ @@ -80,7 +80,7 @@ export function createSeedanceStatusEndpoint(baseUrl: string | undefined, extern if ((base.protocol !== "http:" && base.protocol !== "https:") || base.username || base.password || base.search || base.hash) { return undefined; } - return `${baseUrl.replace(/\/+$/, "")}/contents/generations/tasks/${encodeURIComponent(externalOperationId)}`; + return `${stripTrailingSlashes(baseUrl)}/contents/generations/tasks/${encodeURIComponent(externalOperationId)}`; } catch { return undefined; } diff --git a/packages/providers/src/vendors/seedance.ts b/packages/providers/src/vendors/seedance.ts index 5f6c1636f..6afd65f2a 100644 --- a/packages/providers/src/vendors/seedance.ts +++ b/packages/providers/src/vendors/seedance.ts @@ -14,7 +14,7 @@ import type { MediaVendorTransportRequest, MediaVendorTransportResult, } from "../types.js"; -import { isRecord, removeUndefined } from "../utils.js"; +import { isRecord, removeUndefined, stripTrailingSlashes } from "../utils.js"; import { createSeedanceStatusEndpoint, decodeSeedanceTaskStatusResponse, type SeedanceTerminalObservation } from "./seedance-task-status.js"; const MAX_MEDIA_VENDOR_VIDEO_BYTES = 512 * 1024 * 1024; @@ -42,7 +42,7 @@ export async function submitSeedanceVideoTask( request: MediaVendorTransportRequest, options: LiveMediaVendorTransportOptions, ): Promise { - const endpoint = `${options.baseUrl?.replace(/\/+$/, "")}/contents/generations/tasks`; + const endpoint = `${stripTrailingSlashes(options.baseUrl)}/contents/generations/tasks`; const payload = buildSeedanceVideoTaskPayload(request.requestPlan); const httpClient = options.httpClient ?? defaultMediaVendorHttpClient; const abortController = new AbortController(); @@ -226,7 +226,7 @@ export async function executeSeedanceVideoTask( request: MediaVendorTransportRequest, options: LiveMediaVendorTransportOptions, ): Promise { - const endpoint = `${options.baseUrl?.replace(/\/+$/, "")}/contents/generations/tasks`; + const endpoint = `${stripTrailingSlashes(options.baseUrl)}/contents/generations/tasks`; const payload = buildSeedanceVideoTaskPayload(request.requestPlan); const httpClient = options.httpClient ?? defaultMediaVendorHttpClient; const abortController = new AbortController(); diff --git a/packages/providers/src/vendors/seedream.ts b/packages/providers/src/vendors/seedream.ts index 621f96513..cbdce66eb 100644 --- a/packages/providers/src/vendors/seedream.ts +++ b/packages/providers/src/vendors/seedream.ts @@ -10,7 +10,7 @@ import type { MediaVendorTransportRequest, MediaVendorTransportResult, } from "../types.js"; -import { isRecord, removeUndefined } from "../utils.js"; +import { isRecord, removeUndefined, stripTrailingSlashes } from "../utils.js"; const MAX_MEDIA_VENDOR_IMAGE_BYTES = 25 * 1024 * 1024; const DEFAULT_LIVE_MEDIA_VENDOR_TIMEOUT_MS = 180_000; @@ -19,7 +19,7 @@ export async function executeSeedreamImageGeneration( request: MediaVendorTransportRequest, options: LiveMediaVendorTransportOptions, ): Promise { - const endpoint = `${options.baseUrl?.replace(/\/+$/, "")}/images/generations`; + const endpoint = `${stripTrailingSlashes(options.baseUrl)}/images/generations`; const payload = buildSeedreamImageGenerationPayload(request.requestPlan); return await executeSeedreamImageRequest({ request, @@ -37,7 +37,7 @@ export async function executeSeedreamImageEdit( request: MediaVendorTransportRequest, options: LiveMediaVendorTransportOptions, ): Promise { - const endpoint = `${options.baseUrl?.replace(/\/+$/, "")}/images/generations`; + const endpoint = `${stripTrailingSlashes(options.baseUrl)}/images/generations`; const payloadResolution = buildSeedreamImageEditPayload(request.requestPlan); if (!payloadResolution.ok) { return { diff --git a/scripts/gstack/testsavant-runtime.mjs b/scripts/gstack/testsavant-runtime.mjs index 9fa45e17e..3ce54c22e 100644 --- a/scripts/gstack/testsavant-runtime.mjs +++ b/scripts/gstack/testsavant-runtime.mjs @@ -22,12 +22,14 @@ export async function verifyTestsavantRuntimeFiles(modelRoot) { } } -// Matches the frozen upstream preprocessing, including its entity order. +// Matches the frozen upstream preprocessing while decoding each entity once. export function testsavantPlainText(input) { if (!input.includes("<")) return input; return input.replace(/<(script|style)[^>]*>[\s\S]*?<\/\1>/gi, " ").replace(/<[^>]+>/g, " ") - .replace(/ /g, " ").replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">") - .replace(/"/g, '"').replace(/\s+/g, " ").trim(); + .replace(/&(?:nbsp|amp|lt|gt|quot);/g, entity => ({ + " ": " ", "&": "&", "<": "<", ">": ">", """: '"', + })[entity] ?? entity) + .replace(/\s+/g, " ").trim(); } export function planTestsavantWindows(ids) { diff --git a/scripts/gstack/testsavant-runtime.test.mjs b/scripts/gstack/testsavant-runtime.test.mjs index e1bca7417..ea544f258 100644 --- a/scripts/gstack/testsavant-runtime.test.mjs +++ b/scripts/gstack/testsavant-runtime.test.mjs @@ -44,6 +44,10 @@ test("platform resolver uses only packaged Node, and child environment excludes API_KEY: "not-real", PATH: "evil", HOME: "/empty", SystemRoot: "C:\\Windows" }), { HOME: "/empty", SystemRoot: "C:\\Windows" }); }); +test("TestSavant plain text decoding preserves nested entities for a later layer", () => { + assert.equal(testsavantPlainText("

&lt; <

"), "< <"); +}); + test("the actual gs route receives its vocabulary, without changing the upstream combiner thresholds", () => { for (const [input, output] of [["safe", "safe"], ["log_only", "safe"], ["warn", "suspicious"], ["block", "unsafe"]]) { assert.equal(testsavantRouteVerdict({ verdict: input }), output);