Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 21 additions & 3 deletions packages/ai/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,15 +36,33 @@ const program = Effect.gen(function* () {
const response = yield* Image.generate({
model: OpenAI.configure({ apiKey: process.env.OPENAI_API_KEY }).image("gpt-image-2"),
prompt: "A robot tending a rooftop garden",
count: 2,
size: { width: 1024, height: 1024 },
providerOptions: { openai: { quality: "high", outputFormat: "webp" } },
options: {
n: 2,
size: "1024x1024",
quality: "high", // inferred from the OpenAI image model
outputFormat: "webp",
future_option: true, // unknown native options pass through unchanged
},
})

return response.images // GeneratedImage[] with owned bytes or a provider URL
})
```

Provider-native image options belong to each request. Raw `http.body` fields have final precedence over them:

```ts
const model = OpenAI.configure({ apiKey }).image("gpt-image-2")

yield *
Image.generate({
model,
prompt,
options: { quality: "medium" },
http,
})
```

Conversational image generation remains part of the LLM interaction. OpenAI Responses exposes it through its hosted image tool:

```ts
Expand Down
2 changes: 1 addition & 1 deletion packages/ai/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
"scripts": {
"setup:recording-env": "bun run script/setup-recording-env.ts",
"test": "bun test --timeout 30000 --only-failures",
"typecheck": "tsgo --noEmit",
"typecheck": "tsgo --noEmit && tsgo --noEmit -p tsconfig.types.json",
"build": "tsc -p tsconfig.build.json"
},
"files": [
Expand Down
10 changes: 7 additions & 3 deletions packages/ai/src/image-client.ts
Original file line number Diff line number Diff line change
@@ -1,17 +1,21 @@
import { Context, Effect, Layer } from "effect"
import { RequestExecutor } from "./route/executor"
import type { ImageRequest, ImageResponse } from "./image"
import type { ImageOptions, ImageRequest, ImageRequestFor, ImageResponse } from "./image"
import type { LLMError } from "./schema"

export type Execute = RequestExecutor.Interface["execute"]

export interface Interface {
readonly generate: (request: ImageRequest) => Effect.Effect<ImageResponse, LLMError>
readonly generate: <Options extends ImageOptions>(
request: ImageRequestFor<Options>,
) => Effect.Effect<ImageResponse, LLMError>
}

export class Service extends Context.Service<Service, Interface>()("@opencode/ImageClient") {}

export const generate = (request: ImageRequest): Effect.Effect<ImageResponse, LLMError> =>
export const generate = <Options extends ImageOptions>(
request: ImageRequestFor<Options>,
): Effect.Effect<ImageResponse, LLMError> =>
Effect.gen(function* () {
const client = yield* Service
return yield* client.generate(request)
Expand Down
95 changes: 55 additions & 40 deletions packages/ai/src/image.ts
Original file line number Diff line number Diff line change
@@ -1,80 +1,85 @@
import { Effect, Schema } from "effect"
import { HttpOptions, InvalidRequestReason, LLMError, ModelID, ProviderID, ProviderMetadata, Usage } from "./schema"
import { ImageClient, type Execute as ImageExecute } from "./image-client"
import { ImageClient, Service, type Execute as ImageExecute } from "./image-client"

export interface ImageRoute {
export interface ImageRoute<Options extends ImageOptions = ImageOptions> {
readonly id: string
readonly generate: (request: ImageRequest, execute: ImageExecute) => Effect.Effect<ImageResponse, LLMError>
readonly generate: (
request: ImageRequestFor<Options>,
execute: ImageExecute,
) => Effect.Effect<ImageResponse, LLMError>
}

export class ImageModel {
export type ImageOptions = Record<string, unknown>

export class ImageModel<Options extends ImageOptions = ImageOptions> {
declare protected readonly _Options: (options: Options) => Options
readonly id: ModelID
readonly provider: ProviderID
readonly route: ImageRoute
readonly defaults?: ImageModelDefaults
readonly route: ImageRoute<Options>
readonly http?: HttpOptions

constructor(input: ImageModel.Input) {
constructor(input: ImageModel.Input<Options>) {
this.id = input.id
this.provider = input.provider
this.route = input.route
this.defaults = input.defaults
this.http = input.http
}

static make(input: ImageModel.MakeInput) {
return new ImageModel({
static make<Options extends ImageOptions = ImageOptions>(input: ImageModel.MakeInput<Options>) {
return new ImageModel<Options>({
id: ModelID.make(input.id),
provider: ProviderID.make(input.provider),
route: input.route,
defaults: input.defaults,
http: input.http,
})
}
}

export namespace ImageModel {
export interface Input {
export interface Input<Options extends ImageOptions = ImageOptions> {
readonly id: ModelID
readonly provider: ProviderID
readonly route: ImageRoute
readonly defaults?: ImageModelDefaults
readonly route: ImageRoute<Options>
readonly http?: HttpOptions
}

export interface MakeInput extends Omit<Input, "id" | "provider"> {
export interface MakeInput<Options extends ImageOptions = ImageOptions>
extends Omit<Input<Options>, "id" | "provider"> {
readonly id: string | ModelID
readonly provider: string | ProviderID
}
}

export interface ImageModelDefaults {
readonly providerOptions?: Record<string, Record<string, unknown>>
readonly http?: HttpOptions
}

export const ImageModelSchema = Schema.declare((value): value is ImageModel => value instanceof ImageModel, {
expected: "Image.Model",
})

export const ImageSize = Schema.Struct({
width: Schema.Int.check(Schema.isGreaterThanOrEqualTo(1)),
height: Schema.Int.check(Schema.isGreaterThanOrEqualTo(1)),
}).annotate({ identifier: "Image.Size" })
export type ImageSize = Schema.Schema.Type<typeof ImageSize>

export class ImageRequest extends Schema.Class<ImageRequest>("Image.Request")({
model: ImageModelSchema,
prompt: Schema.String,
count: Schema.optional(Schema.Int.check(Schema.isGreaterThanOrEqualTo(1))),
size: Schema.optional(ImageSize),
aspectRatio: Schema.optional(Schema.String),
seed: Schema.optional(Schema.Number),
providerOptions: Schema.optional(Schema.Record(Schema.String, Schema.Record(Schema.String, Schema.Unknown))),
options: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
http: Schema.optional(HttpOptions),
metadata: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
}) {}
}) {
declare protected readonly _ImageRequest: void
}

export type ImageRequestInput = Omit<ConstructorParameters<typeof ImageRequest>[0], "http"> & {
readonly http?: HttpOptions.Input
export type ImageRequestFor<Options extends ImageOptions = ImageOptions> = Omit<ImageRequest, "model" | "options"> & {
readonly model: ImageModel<Options>
readonly options?: Options
}

export type ImageModelOptions<Model> = Model extends ImageModel<infer Options> ? Options : never

export type ImageRequestInput<Model extends object = ImageModel> = Omit<
ConstructorParameters<typeof ImageRequest>[0],
"model" | "options" | "http"
> & {
readonly model: Model
readonly options?: NoInfer<ImageModelOptions<Model>>
readonly http?: HttpOptions.Input
} & (Model extends ImageModel<ImageModelOptions<Model>> ? unknown : never)

export class GeneratedImage extends Schema.Class<GeneratedImage>("Image.Generated")({
mediaType: Schema.String,
data: Schema.Union([Schema.String, Schema.Uint8Array]),
Expand All @@ -91,24 +96,34 @@ export class ImageResponse extends Schema.Class<ImageResponse>("Image.Response")
}
}

export const request = (input: ImageRequest | ImageRequestInput) => {
export function request<const Model extends object>(
input: ImageRequestInput<Model>,
): ImageRequestFor<ImageModelOptions<Model>>
export function request(input: ImageRequest): ImageRequest
export function request(input: ImageRequest | ImageRequestInput) {
if (input instanceof ImageRequest) return input
return new ImageRequest({
...input,
model: input.model as unknown as ImageModel,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
}

export const generate = (input: ImageRequest | ImageRequestInput) =>
Effect.try({
try: () => request(input),
export function generate<const Model extends object>(
input: ImageRequestInput<Model>,
): Effect.Effect<ImageResponse, LLMError, Service>
export function generate(input: ImageRequest): Effect.Effect<ImageResponse, LLMError, Service>
export function generate(input: ImageRequest | ImageRequestInput) {
return Effect.try({
try: () => (input instanceof ImageRequest ? input : request(input)),
catch: (error) =>
new LLMError({
module: "Image",
method: "generate",
reason: new InvalidRequestReason({ message: error instanceof Error ? error.message : String(error) }),
}),
}).pipe(Effect.flatMap(ImageClient.generate))
}).pipe(Effect.flatMap((request) => ImageClient.generate(request as unknown as ImageRequestFor<ImageOptions>)))
}

export const Image = {
request,
Expand Down
4 changes: 2 additions & 2 deletions packages/ai/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@ export type {
Service as LLMClientService,
} from "./route/client"
export * from "./schema"
export { GeneratedImage, ImageModel, ImageRequest, ImageResponse, ImageSize } from "./image"
export type { ImageModelDefaults, ImageRequestInput, ImageRoute } from "./image"
export { GeneratedImage, ImageModel, ImageRequest, ImageResponse } from "./image"
export type { ImageModelOptions, ImageOptions, ImageRequestFor, ImageRequestInput, ImageRoute } from "./image"
export { Image } from "./image"
export { Tool, ToolFailure, toDefinitions } from "./tool"
export { ToolRuntime } from "./tool-runtime"
Expand Down
Loading
Loading