Skip to content

Commit 4cbf9ab

Browse files
Copilotmudler
Copilot
and
mudler
authored
feat: Add Anthropic Messages API support (#7948)
* Initial plan * Add Anthropic Messages API support Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> * Fix code review comments: add error handling for JSON operations Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> * Fix test suite to use existing schema test runner Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> * Add Anthropic e2e tests using anthropic-sdk-go for streaming and non-streaming Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> * Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com>
1 parent fdc2c07 commit 4cbf9ab

8 files changed

Lines changed: 887 additions & 1 deletion

File tree

‎core/http/app.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,7 @@ func API(application *application.Application) (*echo.Echo, error) {
205205

206206
routes.RegisterLocalAIRoutes(e, requestExtractor, application.ModelConfigLoader(), application.ModelLoader(), application.ApplicationConfig(), application.GalleryService(), opcache, application.TemplatesEvaluator(), application)
207207
routes.RegisterOpenAIRoutes(e, requestExtractor, application)
208+
routes.RegisterAnthropicRoutes(e, requestExtractor, application)
208209
if !application.ApplicationConfig().DisableWebUI {
209210
routes.RegisterUIAPIRoutes(e, application.ModelConfigLoader(), application.ModelLoader(), application.ApplicationConfig(), application.GalleryService(), opcache, application)
210211
routes.RegisterUIRoutes(e, application.ModelConfigLoader(), application.ModelLoader(), application.ApplicationConfig(), application.GalleryService())

‎core/http/endpoints/anthropic/messages.go‎

Lines changed: 310 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,310 @@
1+
package anthropic
2+
3+
import (
4+
"encoding/json"
5+
"fmt"
6+
7+
"github.com/google/uuid"
8+
"github.com/labstack/echo/v4"
9+
"github.com/mudler/LocalAI/core/backend"
10+
"github.com/mudler/LocalAI/core/config"
11+
"github.com/mudler/LocalAI/core/http/middleware"
12+
"github.com/mudler/LocalAI/core/schema"
13+
"github.com/mudler/LocalAI/core/templates"
14+
"github.com/mudler/LocalAI/pkg/model"
15+
"github.com/mudler/xlog"
16+
)
17+
18+
// MessagesEndpoint is the Anthropic Messages API endpoint
19+
// https://docs.anthropic.com/claude/reference/messages_post
20+
// @Summary Generate a message response for the given messages and model.
21+
// @Param request body schema.AnthropicRequest true "query params"
22+
// @Success 200 {object} schema.AnthropicResponse "Response"
23+
// @Router /v1/messages [post]
24+
func MessagesEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator *templates.Evaluator, appConfig *config.ApplicationConfig) echo.HandlerFunc {
25+
return func(c echo.Context) error {
26+
id := uuid.New().String()
27+
28+
input, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST).(*schema.AnthropicRequest)
29+
if !ok || input.Model == "" {
30+
return sendAnthropicError(c, 400, "invalid_request_error", "model is required")
31+
}
32+
33+
cfg, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG).(*config.ModelConfig)
34+
if !ok || cfg == nil {
35+
return sendAnthropicError(c, 400, "invalid_request_error", "model configuration not found")
36+
}
37+
38+
if input.MaxTokens <= 0 {
39+
return sendAnthropicError(c, 400, "invalid_request_error", "max_tokens is required and must be greater than 0")
40+
}
41+
42+
xlog.Debug("Anthropic Messages endpoint configuration read", "config", cfg)
43+
44+
// Convert Anthropic messages to OpenAI format for internal processing
45+
openAIMessages := convertAnthropicToOpenAIMessages(input)
46+
47+
// Create an OpenAI-compatible request for internal processing
48+
openAIReq := &schema.OpenAIRequest{
49+
PredictionOptions: schema.PredictionOptions{
50+
BasicModelRequest: schema.BasicModelRequest{Model: input.Model},
51+
Temperature: input.Temperature,
52+
TopK: input.TopK,
53+
TopP: input.TopP,
54+
Maxtokens: &input.MaxTokens,
55+
},
56+
Messages: openAIMessages,
57+
Stream: input.Stream,
58+
Context: input.Context,
59+
Cancel: input.Cancel,
60+
}
61+
62+
// Set stop sequences
63+
if len(input.StopSequences) > 0 {
64+
openAIReq.Stop = input.StopSequences
65+
}
66+
67+
// Merge config settings
68+
if input.Temperature != nil {
69+
cfg.Temperature = input.Temperature
70+
}
71+
if input.TopK != nil {
72+
cfg.TopK = input.TopK
73+
}
74+
if input.TopP != nil {
75+
cfg.TopP = input.TopP
76+
}
77+
cfg.Maxtokens = &input.MaxTokens
78+
if len(input.StopSequences) > 0 {
79+
cfg.StopWords = append(cfg.StopWords, input.StopSequences...)
80+
}
81+
82+
// Template the prompt
83+
predInput := evaluator.TemplateMessages(*openAIReq, openAIReq.Messages, cfg, nil, false)
84+
xlog.Debug("Anthropic Messages - Prompt (after templating)", "prompt", predInput)
85+
86+
if input.Stream {
87+
return handleAnthropicStream(c, id, input, cfg, ml, predInput)
88+
}
89+
90+
return handleAnthropicNonStream(c, id, input, cfg, ml, predInput, openAIReq)
91+
}
92+
}
93+
94+
func handleAnthropicNonStream(c echo.Context, id string, input *schema.AnthropicRequest, cfg *config.ModelConfig, ml *model.ModelLoader, predInput string, openAIReq *schema.OpenAIRequest) error {
95+
images := []string{}
96+
for _, m := range openAIReq.Messages {
97+
images = append(images, m.StringImages...)
98+
}
99+
100+
predFunc, err := backend.ModelInference(
101+
input.Context, predInput, openAIReq.Messages, images, nil, nil, ml, cfg, nil, nil, nil, "", "", nil, nil, nil)
102+
if err != nil {
103+
xlog.Error("Anthropic model inference failed", "error", err)
104+
return sendAnthropicError(c, 500, "api_error", fmt.Sprintf("model inference failed: %v", err))
105+
}
106+
107+
prediction, err := predFunc()
108+
if err != nil {
109+
xlog.Error("Anthropic prediction failed", "error", err)
110+
return sendAnthropicError(c, 500, "api_error", fmt.Sprintf("prediction failed: %v", err))
111+
}
112+
113+
result := backend.Finetune(*cfg, predInput, prediction.Response)
114+
stopReason := "end_turn"
115+
116+
resp := &schema.AnthropicResponse{
117+
ID: fmt.Sprintf("msg_%s", id),
118+
Type: "message",
119+
Role: "assistant",
120+
Model: input.Model,
121+
StopReason: &stopReason,
122+
Content: []schema.AnthropicContentBlock{
123+
{Type: "text", Text: result},
124+
},
125+
Usage: schema.AnthropicUsage{
126+
InputTokens: prediction.Usage.Prompt,
127+
OutputTokens: prediction.Usage.Completion,
128+
},
129+
}
130+
131+
if respData, err := json.Marshal(resp); err == nil {
132+
xlog.Debug("Anthropic Response", "response", string(respData))
133+
}
134+
135+
return c.JSON(200, resp)
136+
}
137+
138+
func handleAnthropicStream(c echo.Context, id string, input *schema.AnthropicRequest, cfg *config.ModelConfig, ml *model.ModelLoader, predInput string) error {
139+
c.Response().Header().Set("Content-Type", "text/event-stream")
140+
c.Response().Header().Set("Cache-Control", "no-cache")
141+
c.Response().Header().Set("Connection", "keep-alive")
142+
143+
// Create OpenAI messages for inference
144+
openAIMessages := convertAnthropicToOpenAIMessages(input)
145+
146+
images := []string{}
147+
for _, m := range openAIMessages {
148+
images = append(images, m.StringImages...)
149+
}
150+
151+
// Send message_start event
152+
messageStart := schema.AnthropicStreamEvent{
153+
Type: "message_start",
154+
Message: &schema.AnthropicStreamMessage{
155+
ID: fmt.Sprintf("msg_%s", id),
156+
Type: "message",
157+
Role: "assistant",
158+
Content: []schema.AnthropicContentBlock{},
159+
Model: input.Model,
160+
Usage: schema.AnthropicUsage{InputTokens: 0, OutputTokens: 0},
161+
},
162+
}
163+
sendAnthropicSSE(c, messageStart)
164+
165+
// Send content_block_start event
166+
contentBlockStart := schema.AnthropicStreamEvent{
167+
Type: "content_block_start",
168+
Index: 0,
169+
ContentBlock: &schema.AnthropicContentBlock{Type: "text", Text: ""},
170+
}
171+
sendAnthropicSSE(c, contentBlockStart)
172+
173+
// Stream content deltas
174+
tokenCallback := func(token string, usage backend.TokenUsage) bool {
175+
delta := schema.AnthropicStreamEvent{
176+
Type: "content_block_delta",
177+
Index: 0,
178+
Delta: &schema.AnthropicStreamDelta{
179+
Type: "text_delta",
180+
Text: token,
181+
},
182+
}
183+
sendAnthropicSSE(c, delta)
184+
return true
185+
}
186+
187+
predFunc, err := backend.ModelInference(
188+
input.Context, predInput, openAIMessages, images, nil, nil, ml, cfg, nil, nil, tokenCallback, "", "", nil, nil, nil)
189+
if err != nil {
190+
xlog.Error("Anthropic stream model inference failed", "error", err)
191+
return sendAnthropicError(c, 500, "api_error", fmt.Sprintf("model inference failed: %v", err))
192+
}
193+
194+
prediction, err := predFunc()
195+
if err != nil {
196+
xlog.Error("Anthropic stream prediction failed", "error", err)
197+
return sendAnthropicError(c, 500, "api_error", fmt.Sprintf("prediction failed: %v", err))
198+
}
199+
200+
// Send content_block_stop event
201+
contentBlockStop := schema.AnthropicStreamEvent{
202+
Type: "content_block_stop",
203+
Index: 0,
204+
}
205+
sendAnthropicSSE(c, contentBlockStop)
206+
207+
// Send message_delta event with stop_reason
208+
stopReason := "end_turn"
209+
messageDelta := schema.AnthropicStreamEvent{
210+
Type: "message_delta",
211+
Delta: &schema.AnthropicStreamDelta{
212+
StopReason: &stopReason,
213+
},
214+
Usage: &schema.AnthropicUsage{
215+
OutputTokens: prediction.Usage.Completion,
216+
},
217+
}
218+
sendAnthropicSSE(c, messageDelta)
219+
220+
// Send message_stop event
221+
messageStop := schema.AnthropicStreamEvent{
222+
Type: "message_stop",
223+
}
224+
sendAnthropicSSE(c, messageStop)
225+
226+
return nil
227+
}
228+
229+
func sendAnthropicSSE(c echo.Context, event schema.AnthropicStreamEvent) {
230+
data, err := json.Marshal(event)
231+
if err != nil {
232+
xlog.Error("Failed to marshal SSE event", "error", err)
233+
return
234+
}
235+
fmt.Fprintf(c.Response().Writer, "event: %s\ndata: %s\n\n", event.Type, string(data))
236+
c.Response().Flush()
237+
}
238+
239+
func sendAnthropicError(c echo.Context, statusCode int, errorType, message string) error {
240+
resp := schema.AnthropicErrorResponse{
241+
Type: "error",
242+
Error: schema.AnthropicError{
243+
Type: errorType,
244+
Message: message,
245+
},
246+
}
247+
return c.JSON(statusCode, resp)
248+
}
249+
250+
func convertAnthropicToOpenAIMessages(input *schema.AnthropicRequest) []schema.Message {
251+
var messages []schema.Message
252+
253+
// Add system message if present
254+
if input.System != "" {
255+
messages = append(messages, schema.Message{
256+
Role: "system",
257+
StringContent: input.System,
258+
Content: input.System,
259+
})
260+
}
261+
262+
// Convert Anthropic messages to OpenAI format
263+
for _, msg := range input.Messages {
264+
openAIMsg := schema.Message{
265+
Role: msg.Role,
266+
}
267+
268+
// Handle content (can be string or array of content blocks)
269+
switch content := msg.Content.(type) {
270+
case string:
271+
openAIMsg.StringContent = content
272+
openAIMsg.Content = content
273+
case []interface{}:
274+
// Handle array of content blocks
275+
var textContent string
276+
var stringImages []string
277+
278+
for _, block := range content {
279+
if blockMap, ok := block.(map[string]interface{}); ok {
280+
blockType, _ := blockMap["type"].(string)
281+
switch blockType {
282+
case "text":
283+
if text, ok := blockMap["text"].(string); ok {
284+
textContent += text
285+
}
286+
case "image":
287+
// Handle image content
288+
if source, ok := blockMap["source"].(map[string]interface{}); ok {
289+
if sourceType, ok := source["type"].(string); ok && sourceType == "base64" {
290+
if data, ok := source["data"].(string); ok {
291+
mediaType, _ := source["media_type"].(string)
292+
// Format as data URI
293+
dataURI := fmt.Sprintf("data:%s;base64,%s", mediaType, data)
294+
stringImages = append(stringImages, dataURI)
295+
}
296+
}
297+
}
298+
}
299+
}
300+
}
301+
openAIMsg.StringContent = textContent
302+
openAIMsg.Content = textContent
303+
openAIMsg.StringImages = stringImages
304+
}
305+
306+
messages = append(messages, openAIMsg)
307+
}
308+
309+
return messages
310+
}

0 commit comments

Comments
 (0)