Skip to content

Commit 8de71c0

Browse files
eimexdeveimexdev
committed
fix(server): project voice notes for legacy clients
1 parent f67a2f9 commit 8de71c0

9 files changed

Lines changed: 345 additions & 9 deletions

File tree

‎apps/server/src/cli/project.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -313,6 +313,7 @@ const fetchLiveOrchestrationSnapshot = (origin: string, bearerToken: string) =>
313313
const client = yield* makeLiveServerClient(origin);
314314
return yield* client.orchestration.snapshot({
315315
headers: { authorization: `Bearer ${bearerToken}` },
316+
query: { audioAttachments: "true" },
316317
});
317318
}).pipe(
318319
withProjectCliLiveServerTimeout,

‎apps/server/src/orchestration/clientCompatibility.test.ts‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
import { describe, expect, it } from "vite-plus/test";
2+
import {
3+
EventId,
4+
MessageId,
5+
ThreadId,
6+
type OrchestrationMessage,
7+
type ThreadMessageSentPayload,
8+
} from "@t3tools/contracts";
9+
10+
import {
11+
projectOrchestrationEventForClient,
12+
projectOrchestrationMessageForClient,
13+
} from "./clientCompatibility.ts";
14+
15+
const audioAttachment = {
16+
type: "audio" as const,
17+
id: "audio-1",
18+
name: "Voice note",
19+
mimeType: "audio/webm",
20+
sizeBytes: 128,
21+
durationMs: 1_000,
22+
waveform: [0.1, 0.8],
23+
};
24+
25+
const message: OrchestrationMessage = {
26+
id: MessageId.make("message-1"),
27+
role: "user",
28+
text: "",
29+
attachments: [audioAttachment],
30+
turnId: null,
31+
streaming: false,
32+
createdAt: "2026-07-24T12:00:00.000Z",
33+
updatedAt: "2026-07-24T12:00:00.000Z",
34+
};
35+
36+
describe("clientCompatibility", () => {
37+
it("keeps audio attachments for capable clients", () => {
38+
expect(
39+
projectOrchestrationMessageForClient(message, {
40+
audioAttachments: true,
41+
}),
42+
).toBe(message);
43+
});
44+
45+
it("replaces audio attachments with legacy-safe text when capability is missing", () => {
46+
const projected = projectOrchestrationMessageForClient(message, undefined);
47+
const { attachments: _attachments, ...messageWithoutAttachments } = message;
48+
expect(projected).toEqual({
49+
...messageWithoutAttachments,
50+
text: "[Voice note]",
51+
});
52+
expect(projected).not.toHaveProperty("attachments");
53+
});
54+
55+
it("retains supported attachments and an embedded transcript", () => {
56+
expect(
57+
projectOrchestrationMessageForClient(
58+
{
59+
...message,
60+
text: "Additional context",
61+
attachments: [
62+
{
63+
type: "image",
64+
id: "image-1",
65+
name: "reference.png",
66+
mimeType: "image/png",
67+
sizeBytes: 64,
68+
},
69+
{
70+
...audioAttachment,
71+
transcript: "Call Morgan tomorrow.",
72+
transcriptionStatus: "ready",
73+
},
74+
],
75+
},
76+
undefined,
77+
),
78+
).toMatchObject({
79+
text: "Additional context\n\n[Voice note transcript]\nCall Morgan tomorrow.",
80+
attachments: [{ type: "image", id: "image-1" }],
81+
});
82+
});
83+
84+
it("projects live message events for legacy clients", () => {
85+
const payload: ThreadMessageSentPayload = {
86+
threadId: ThreadId.make("thread-1"),
87+
messageId: MessageId.make("message-1"),
88+
role: "user",
89+
text: "",
90+
attachments: [audioAttachment],
91+
turnId: null,
92+
streaming: false,
93+
createdAt: "2026-07-24T12:00:00.000Z",
94+
updatedAt: "2026-07-24T12:00:00.000Z",
95+
};
96+
const event = {
97+
sequence: 1,
98+
eventId: EventId.make("event-1"),
99+
aggregateKind: "thread" as const,
100+
aggregateId: ThreadId.make("thread-1"),
101+
occurredAt: "2026-07-24T12:00:00.000Z",
102+
commandId: null,
103+
causationEventId: null,
104+
correlationId: null,
105+
metadata: {},
106+
type: "thread.message-sent" as const,
107+
payload,
108+
};
109+
110+
expect(projectOrchestrationEventForClient(event, undefined)).toMatchObject({
111+
payload: {
112+
text: "[Voice note]",
113+
},
114+
});
115+
expect(projectOrchestrationEventForClient(event, undefined).payload).not.toHaveProperty(
116+
"attachments",
117+
);
118+
});
119+
});

‎apps/server/src/orchestration/clientCompatibility.ts‎

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
import type {
2+
ChatAttachment,
3+
OrchestrationClientCapabilities,
4+
OrchestrationEvent,
5+
OrchestrationMessage,
6+
OrchestrationReadModel,
7+
OrchestrationThread,
8+
OrchestrationThreadDetailSnapshot,
9+
OrchestrationThreadStreamItem,
10+
ThreadMessageSentPayload,
11+
} from "@t3tools/contracts";
12+
13+
const LEGACY_VOICE_NOTE_PLACEHOLDER = "[Voice note]";
14+
15+
function supportsAudioAttachments(
16+
capabilities: OrchestrationClientCapabilities | undefined,
17+
): boolean {
18+
return capabilities?.audioAttachments === true;
19+
}
20+
21+
function legacyVoiceNoteText(attachment: Extract<ChatAttachment, { type: "audio" }>): string {
22+
const transcript = attachment.transcript?.trim();
23+
return transcript ? `[Voice note transcript]\n${transcript}` : LEGACY_VOICE_NOTE_PLACEHOLDER;
24+
}
25+
26+
function projectMessageFields(input: {
27+
readonly text: string;
28+
readonly attachments?: ReadonlyArray<ChatAttachment> | undefined;
29+
}): {
30+
readonly changed: boolean;
31+
readonly text: string;
32+
readonly attachments?: ReadonlyArray<ChatAttachment>;
33+
} {
34+
const attachments = input.attachments ?? [];
35+
const audioAttachments = attachments.filter(
36+
(attachment): attachment is Extract<ChatAttachment, { type: "audio" }> =>
37+
attachment.type === "audio",
38+
);
39+
if (audioAttachments.length === 0) {
40+
return {
41+
changed: false,
42+
text: input.text,
43+
...(input.attachments ? { attachments: input.attachments } : {}),
44+
};
45+
}
46+
47+
const supportedAttachments = attachments.filter((attachment) => attachment.type !== "audio");
48+
const voiceNoteText = audioAttachments.map(legacyVoiceNoteText).join("\n\n");
49+
const text = input.text.trim().length > 0 ? `${input.text}\n\n${voiceNoteText}` : voiceNoteText;
50+
return {
51+
changed: true,
52+
text,
53+
...(supportedAttachments.length > 0 ? { attachments: supportedAttachments } : {}),
54+
};
55+
}
56+
57+
export function projectOrchestrationMessageForClient(
58+
message: OrchestrationMessage,
59+
capabilities: OrchestrationClientCapabilities | undefined,
60+
): OrchestrationMessage {
61+
if (supportsAudioAttachments(capabilities)) {
62+
return message;
63+
}
64+
const projected = projectMessageFields(message);
65+
if (!projected.changed) {
66+
return message;
67+
}
68+
const { attachments: _attachments, ...messageWithoutAttachments } = message;
69+
return {
70+
...messageWithoutAttachments,
71+
text: projected.text,
72+
...(projected.attachments ? { attachments: [...projected.attachments] } : {}),
73+
};
74+
}
75+
76+
function projectThreadMessageEventForClient(
77+
payload: ThreadMessageSentPayload,
78+
): ThreadMessageSentPayload {
79+
const projected = projectMessageFields(payload);
80+
if (!projected.changed) {
81+
return payload;
82+
}
83+
const { attachments: _attachments, ...payloadWithoutAttachments } = payload;
84+
return {
85+
...payloadWithoutAttachments,
86+
text: projected.text,
87+
...(projected.attachments ? { attachments: [...projected.attachments] } : {}),
88+
};
89+
}
90+
91+
export function projectOrchestrationThreadForClient(
92+
thread: OrchestrationThread,
93+
capabilities: OrchestrationClientCapabilities | undefined,
94+
): OrchestrationThread {
95+
if (supportsAudioAttachments(capabilities)) {
96+
return thread;
97+
}
98+
return {
99+
...thread,
100+
messages: thread.messages.map((message) =>
101+
projectOrchestrationMessageForClient(message, capabilities),
102+
),
103+
};
104+
}
105+
106+
export function projectOrchestrationThreadSnapshotForClient(
107+
snapshot: OrchestrationThreadDetailSnapshot,
108+
capabilities: OrchestrationClientCapabilities | undefined,
109+
): OrchestrationThreadDetailSnapshot {
110+
if (supportsAudioAttachments(capabilities)) {
111+
return snapshot;
112+
}
113+
return {
114+
...snapshot,
115+
thread: projectOrchestrationThreadForClient(snapshot.thread, capabilities),
116+
};
117+
}
118+
119+
export function projectOrchestrationReadModelForClient(
120+
snapshot: OrchestrationReadModel,
121+
capabilities: OrchestrationClientCapabilities | undefined,
122+
): OrchestrationReadModel {
123+
if (supportsAudioAttachments(capabilities)) {
124+
return snapshot;
125+
}
126+
return {
127+
...snapshot,
128+
threads: snapshot.threads.map((thread) =>
129+
projectOrchestrationThreadForClient(thread, capabilities),
130+
),
131+
};
132+
}
133+
134+
export function projectOrchestrationEventForClient(
135+
event: OrchestrationEvent,
136+
capabilities: OrchestrationClientCapabilities | undefined,
137+
): OrchestrationEvent {
138+
if (supportsAudioAttachments(capabilities) || event.type !== "thread.message-sent") {
139+
return event;
140+
}
141+
return {
142+
...event,
143+
payload: projectThreadMessageEventForClient(event.payload),
144+
};
145+
}
146+
147+
export function projectOrchestrationThreadStreamItemForClient(
148+
item: OrchestrationThreadStreamItem,
149+
capabilities: OrchestrationClientCapabilities | undefined,
150+
): OrchestrationThreadStreamItem {
151+
switch (item.kind) {
152+
case "synchronized":
153+
return item;
154+
case "snapshot":
155+
return {
156+
...item,
157+
snapshot: projectOrchestrationThreadSnapshotForClient(item.snapshot, capabilities),
158+
};
159+
case "event":
160+
return {
161+
...item,
162+
event: projectOrchestrationEventForClient(item.event, capabilities),
163+
};
164+
default:
165+
item satisfies never;
166+
return item;
167+
}
168+
}

‎apps/server/src/orchestration/http.ts‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,10 @@ import {
1717
} from "../auth/http.ts";
1818
import { OrchestrationEngineService } from "./Services/OrchestrationEngine.ts";
1919
import { ProjectionSnapshotQuery } from "./Services/ProjectionSnapshotQuery.ts";
20+
import {
21+
projectOrchestrationReadModelForClient,
22+
projectOrchestrationThreadSnapshotForClient,
23+
} from "./clientCompatibility.ts";
2024

2125
export const orchestrationHttpApiLayer = HttpApiBuilder.group(
2226
EnvironmentHttpApi,
@@ -31,13 +35,17 @@ export const orchestrationHttpApiLayer = HttpApiBuilder.group(
3135
Effect.fn("environment.orchestration.snapshot")(function* (args) {
3236
yield* annotateEnvironmentRequest(args.endpoint.name);
3337
yield* requireEnvironmentScope(AuthOrchestrationReadScope);
34-
return yield* projectionSnapshotQuery
35-
.getSnapshot()
36-
.pipe(
37-
Effect.catch((cause) =>
38-
failEnvironmentInternal("orchestration_snapshot_failed", cause),
38+
return yield* projectionSnapshotQuery.getSnapshot().pipe(
39+
Effect.map((snapshot) =>
40+
projectOrchestrationReadModelForClient(
41+
snapshot,
42+
args.query.audioAttachments === "true" ? { audioAttachments: true } : undefined,
3943
),
40-
);
44+
),
45+
Effect.catch((cause) =>
46+
failEnvironmentInternal("orchestration_snapshot_failed", cause),
47+
),
48+
);
4149
}),
4250
)
4351
.handle(
@@ -69,7 +77,10 @@ export const orchestrationHttpApiLayer = HttpApiBuilder.group(
6977
if (Option.isNone(snapshot)) {
7078
return yield* failEnvironmentNotFound("thread_not_found");
7179
}
72-
return snapshot.value;
80+
return projectOrchestrationThreadSnapshotForClient(
81+
snapshot.value,
82+
args.query.audioAttachments === "true" ? { audioAttachments: true } : undefined,
83+
);
7384
}),
7485
)
7586
.handle(

‎apps/server/src/ws.ts‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,10 @@ import * as ServerConfig from "./config.ts";
7070
import * as Keybindings from "./keybindings.ts";
7171
import * as ExternalLauncher from "./process/externalLauncher.ts";
7272
import { normalizeDispatchCommand } from "./orchestration/Normalizer.ts";
73+
import {
74+
projectOrchestrationEventForClient,
75+
projectOrchestrationThreadStreamItemForClient,
76+
} from "./orchestration/clientCompatibility.ts";
7377
import * as OrchestrationEngine from "./orchestration/Services/OrchestrationEngine.ts";
7478
import * as ProjectionSnapshotQuery from "./orchestration/Services/ProjectionSnapshotQuery.ts";
7579
import {
@@ -1227,6 +1231,11 @@ const makeWsRpcLayer = (
12271231
).pipe(
12281232
Effect.map((events) => Array.from(events)),
12291233
Effect.flatMap(enrichOrchestrationEvents),
1234+
Effect.map((events) =>
1235+
events.map((event) =>
1236+
projectOrchestrationEventForClient(event, input.clientCapabilities),
1237+
),
1238+
),
12301239
Effect.mapError(
12311240
(cause) =>
12321241
new OrchestrationReplayEventsError({
@@ -1465,7 +1474,15 @@ const makeWsRpcLayer = (
14651474
}),
14661475
afterSnapshot,
14671476
);
1468-
}),
1477+
}).pipe(
1478+
Effect.map((stream) =>
1479+
stream.pipe(
1480+
Stream.map((item) =>
1481+
projectOrchestrationThreadStreamItemForClient(item, input.clientCapabilities),
1482+
),
1483+
),
1484+
),
1485+
),
14691486
{ "rpc.aggregate": "orchestration" },
14701487
),
14711488
[WS_METHODS.serverProbe]: (_input) =>

0 commit comments

Comments
 (0)