Skip to content

Commit e813b2c

Browse files
130rockirt10708
130rockirt
and
10708
authored
fix(kun): route media generation through provider proxy for inline configs (#1269)
Image/speech/music/video tool configs without a providerId never resolve a provider credential, so createProxyFetch received an empty proxyUrl and the requests silently bypassed the configured model-request proxy. Fall back to the provider-level model proxy so custom inline media providers stay on the same proxied path as chat model requests. A resolved connection credential stays authoritative: when the connection explicitly bypasses the app proxy, its empty proxyUrl is honored instead of falling back, matching the chat model request routing. Co-authored-by: 10708 <10708@users.noreply.github.com>
1 parent 69ff6a5 commit e813b2c

5 files changed

Lines changed: 139 additions & 17 deletions

File tree

‎kun/src/adapters/tool/image-gen-tool-provider.ts‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,12 @@ export type ImageGenToolProviderOptions = {
9898
attachmentStore?: AttachmentStore
9999
nowIso?: () => string
100100
resolveCredential?: ProviderCredentialResolver
101+
/**
102+
* Provider-level model proxy. Custom inline configs have no provider
103+
* credential to resolve, so without this fallback their requests would
104+
* bypass the proxy that chat model requests honor.
105+
*/
106+
proxyUrl?: string
101107
}
102108

103109
export type ProviderCredentialResolver = (providerId: string) => Promise<{
@@ -316,13 +322,19 @@ export function buildImageGenToolProviders(
316322
const credential = config.providerId && options.resolveCredential
317323
? await options.resolveCredential(config.providerId)
318324
: undefined
325+
// A resolved connection credential is authoritative: an empty proxyUrl
326+
// means the connection explicitly bypasses the app proxy. Only inline
327+
// configs without a providerId fall back to the provider-level proxy.
328+
const proxyUrl = credential
329+
? credential.proxyUrl?.trim() || ''
330+
: options.proxyUrl?.trim() || ''
319331
client = createImageGenClient({
320332
...config,
321333
...(credential ? {
322334
apiKey: credential.apiKey,
323-
headers: { ...(config.headers ?? {}), ...(credential.headers ?? {}) },
324-
...(credential.proxyUrl ? { proxyUrl: credential.proxyUrl } : {})
325-
} : {})
335+
headers: { ...(config.headers ?? {}), ...(credential.headers ?? {}) }
336+
} : {}),
337+
...(proxyUrl ? { proxyUrl } : {})
326338
})
327339
}
328340
const request = {

‎kun/src/adapters/tool/media-gen-proxy.test.ts‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ const { createSpeechGenClient, createMusicGenClient } = await import('./media-ge
1717
const { createVideoGenClient } = await import('./media-gen-video-clients.js')
1818
const { createMediaFetch } = await import('./media-gen-client-support.js')
1919
const { buildImageGenToolProviders } = await import('./image-gen-tool-provider.js')
20+
const { buildSpeechGenToolProviders } = await import('./media-gen-tool-provider.js')
2021

2122
const fakeGeneratedImage = {
2223
// Smallest detectable PNG payload so detectImage() accepts it and the tool
@@ -128,6 +129,95 @@ describe('media generation proxy fetch wiring', () => {
128129
const clientConfig = createImageGenClientMock.mock.calls[0][0] as Record<string, unknown>
129130
expect(clientConfig).not.toHaveProperty('proxyUrl')
130131
})
132+
133+
it('falls back to the provider-level proxy for inline image configs without a providerId', async () => {
134+
const { providers, available } = buildImageGenToolProviders({
135+
...imageGenConfigDefaults,
136+
enabled: true,
137+
protocol: 'openai-images',
138+
baseUrl: 'https://images.example.test/v1',
139+
apiKey: 'sk-inline',
140+
model: 'test-model'
141+
}, {
142+
proxyUrl: 'http://proxy.lan:8080'
143+
})
144+
145+
expect(available).toBe(true)
146+
const tool = providers[0].tools.find((candidate) => candidate.name === 'generate_image')
147+
const result = await tool!.execute({ prompt: 'a cat' }, minimalContext())
148+
expect(result.isError).toBeFalsy()
149+
150+
const clientConfig = createImageGenClientMock.mock.calls[0][0] as Record<string, unknown>
151+
expect(clientConfig.proxyUrl).toBe('http://proxy.lan:8080')
152+
expect(clientConfig.apiKey).toBe('sk-inline')
153+
})
154+
155+
it('prefers the resolved credential proxy over the provider-level fallback', async () => {
156+
const { providers } = buildImageGenToolProviders({
157+
...imageGenConfigDefaults,
158+
enabled: true,
159+
protocol: 'openai-images',
160+
baseUrl: 'https://images.example.test/v1',
161+
model: 'test-model',
162+
providerId: 'prov-1'
163+
}, {
164+
proxyUrl: 'http://fallback.lan:8080',
165+
resolveCredential: async () => ({
166+
apiKey: 'sk-test',
167+
proxyUrl: 'http://proxy.lan:8080'
168+
})
169+
})
170+
171+
const tool = providers[0].tools.find((candidate) => candidate.name === 'generate_image')
172+
await tool!.execute({ prompt: 'a cat' }, minimalContext())
173+
174+
const clientConfig = createImageGenClientMock.mock.calls[0][0] as Record<string, unknown>
175+
expect(clientConfig.proxyUrl).toBe('http://proxy.lan:8080')
176+
})
177+
178+
it('honors a connection that explicitly bypasses the proxy instead of falling back', async () => {
179+
const { providers } = buildImageGenToolProviders({
180+
...imageGenConfigDefaults,
181+
enabled: true,
182+
protocol: 'openai-images',
183+
baseUrl: 'https://images.example.test/v1',
184+
model: 'test-model',
185+
providerId: 'prov-direct'
186+
}, {
187+
proxyUrl: 'http://fallback.lan:8080',
188+
resolveCredential: async () => ({ apiKey: 'sk-test' })
189+
})
190+
191+
const tool = providers[0].tools.find((candidate) => candidate.name === 'generate_image')
192+
await tool!.execute({ prompt: 'a cat' }, minimalContext())
193+
194+
const clientConfig = createImageGenClientMock.mock.calls[0][0] as Record<string, unknown>
195+
expect(clientConfig).not.toHaveProperty('proxyUrl')
196+
})
197+
198+
it('falls back to the provider-level proxy for inline speech configs without a providerId', async () => {
199+
createProxyFetchMock.mockReturnValue(async () => {
200+
throw new Error('proxied fetch invoked')
201+
})
202+
const { providers, available } = buildSpeechGenToolProviders({
203+
enabled: true,
204+
protocol: 'openai-speech',
205+
baseUrl: 'https://speech.example.test/v1',
206+
apiKey: 'sk-inline',
207+
model: 'tts-test',
208+
format: 'mp3',
209+
timeoutMs: 30_000
210+
}, {
211+
proxyUrl: 'http://proxy.lan:8080'
212+
})
213+
214+
expect(available).toBe(true)
215+
const tool = providers[0].tools.find((candidate) => candidate.name === 'generate_speech')
216+
const result = await tool!.execute({ text: 'hello' }, minimalContext())
217+
expect(result.isError).toBe(true)
218+
219+
expect(createProxyFetchMock).toHaveBeenCalledWith('http://proxy.lan:8080')
220+
})
131221
})
132222

133223
function minimalContext(): Parameters<

‎kun/src/adapters/tool/media-gen-tool-provider.ts‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,12 @@ export type MediaGenToolProviderOptions = {
146146
videoClient?: VideoGenClient
147147
nowIso?: () => string
148148
resolveCredential?: ProviderCredentialResolver
149+
/**
150+
* Provider-level model proxy. Custom inline configs have no provider
151+
* credential to resolve, so without this fallback their requests would
152+
* bypass the proxy that chat model requests honor.
153+
*/
154+
proxyUrl?: string
149155
}
150156

151157
export type SpeechGenToolProviderBuildResult = {
@@ -214,7 +220,7 @@ export function buildSpeechGenToolProviders(
214220
const requestTelemetry = () => telemetry(startedAt, client?.id ?? 'speech-provider')
215221
try {
216222
if (!client) {
217-
client = createSpeechGenClient(await resolveProviderCredential(config, options.resolveCredential))
223+
client = createSpeechGenClient(await resolveProviderCredential(config, options.resolveCredential, options.proxyUrl))
218224
}
219225
const media = await client.generate({
220226
text,
@@ -308,7 +314,7 @@ export function buildMusicGenToolProviders(
308314
const requestTelemetry = () => telemetry(startedAt, client?.id ?? 'music-provider')
309315
try {
310316
if (!client) {
311-
client = createMusicGenClient(await resolveProviderCredential(config, options.resolveCredential))
317+
client = createMusicGenClient(await resolveProviderCredential(config, options.resolveCredential, options.proxyUrl))
312318
}
313319
const media = await client.generate({
314320
...(prompt ? { prompt } : {}),
@@ -447,7 +453,7 @@ export function buildVideoGenToolProviders(
447453
const requestTelemetry = () => telemetry(startedAt, client?.id ?? 'video-provider')
448454
try {
449455
if (!client) {
450-
client = createVideoGenClient(await resolveProviderCredential(config, options.resolveCredential))
456+
client = createVideoGenClient(await resolveProviderCredential(config, options.resolveCredential, options.proxyUrl))
451457
}
452458
const media = await client.generate({
453459
prompt,
@@ -508,18 +514,28 @@ async function resolveProviderCredential<T extends {
508514
providerId?: string
509515
apiKey?: string
510516
headers?: Record<string, string>
511-
}>(config: T, resolveCredential?: ProviderCredentialResolver): Promise<T & {
517+
}>(
518+
config: T,
519+
resolveCredential?: ProviderCredentialResolver,
520+
fallbackProxyUrl?: string
521+
): Promise<T & {
512522
apiKey?: string
513523
headers?: Record<string, string>
514524
proxyUrl?: string
515525
}> {
516-
if (!config.providerId || !resolveCredential) return config
526+
const fallbackProxy = fallbackProxyUrl?.trim() ?? ''
527+
if (!config.providerId || !resolveCredential) {
528+
return fallbackProxy ? { ...config, proxyUrl: fallbackProxy } : config
529+
}
517530
const credential = await resolveCredential(config.providerId)
531+
// A resolved connection credential is authoritative: an empty proxyUrl means
532+
// the connection explicitly bypasses the app proxy, so do not fall back here.
533+
const proxyUrl = credential.proxyUrl?.trim() || ''
518534
return {
519535
...config,
520536
apiKey: credential.apiKey,
521537
headers: { ...(config.headers ?? {}), ...(credential.headers ?? {}) },
522-
...(credential.proxyUrl ? { proxyUrl: credential.proxyUrl } : {})
538+
...(proxyUrl ? { proxyUrl } : {})
523539
}
524540
}
525541

‎kun/src/server/runtime-composition-config.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -310,19 +310,19 @@ export function createRuntimeConfigController(
310310
const nextImageGenProviders = buildImageGenToolProviders(nextOptions.capabilities?.imageGen, {
311311
attachmentStore: nextAttachmentStore,
312312
nowIso,
313-
resolveCredential: resolveCapabilityProviderCredential
313+
resolveCredential: resolveCapabilityProviderCredential, proxyUrl: nextOptions.modelProxyUrl
314314
})
315315
const nextSpeechGenProviders = buildSpeechGenToolProviders(nextOptions.capabilities?.speechGen, {
316316
nowIso,
317-
resolveCredential: resolveCapabilityProviderCredential
317+
resolveCredential: resolveCapabilityProviderCredential, proxyUrl: nextOptions.modelProxyUrl
318318
})
319319
const nextMusicGenProviders = buildMusicGenToolProviders(nextOptions.capabilities?.musicGen, {
320320
nowIso,
321-
resolveCredential: resolveCapabilityProviderCredential
321+
resolveCredential: resolveCapabilityProviderCredential, proxyUrl: nextOptions.modelProxyUrl
322322
})
323323
const nextVideoGenProviders = buildVideoGenToolProviders(nextOptions.capabilities?.videoGen, {
324324
nowIso,
325-
resolveCredential: resolveCapabilityProviderCredential
325+
resolveCredential: resolveCapabilityProviderCredential, proxyUrl: nextOptions.modelProxyUrl
326326
})
327327
const nextComputerUseProviders = await buildComputerUseToolProviders(nextOptions.capabilities?.computerUse)
328328
const nextBrowserUseProviders = buildBrowserUseToolProviders(nextOptions.capabilities?.browserUse)

‎kun/src/server/runtime-composition-services.ts‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -319,19 +319,23 @@ export async function createRuntimeServices(
319319
let imageGenProviders = buildImageGenToolProviders(core.activeOptions.capabilities?.imageGen, {
320320
attachmentStore,
321321
nowIso,
322-
resolveCredential: resolveCapabilityProviderCredential
322+
resolveCredential: resolveCapabilityProviderCredential,
323+
proxyUrl: core.activeOptions.modelProxyUrl
323324
})
324325
let speechGenProviders = buildSpeechGenToolProviders(core.activeOptions.capabilities?.speechGen, {
325326
nowIso,
326-
resolveCredential: resolveCapabilityProviderCredential
327+
resolveCredential: resolveCapabilityProviderCredential,
328+
proxyUrl: core.activeOptions.modelProxyUrl
327329
})
328330
let musicGenProviders = buildMusicGenToolProviders(core.activeOptions.capabilities?.musicGen, {
329331
nowIso,
330-
resolveCredential: resolveCapabilityProviderCredential
332+
resolveCredential: resolveCapabilityProviderCredential,
333+
proxyUrl: core.activeOptions.modelProxyUrl
331334
})
332335
let videoGenProviders = buildVideoGenToolProviders(core.activeOptions.capabilities?.videoGen, {
333336
nowIso,
334-
resolveCredential: resolveCapabilityProviderCredential
337+
resolveCredential: resolveCapabilityProviderCredential,
338+
proxyUrl: core.activeOptions.modelProxyUrl
335339
})
336340
let computerUseProviders = await buildComputerUseToolProviders(core.activeOptions.capabilities?.computerUse)
337341
let browserUseProviders = buildBrowserUseToolProviders(core.activeOptions.capabilities?.browserUse)

0 commit comments

Comments
 (0)