Merge pull request #152 from LianjiaTech/develop

merge to main

Author: szl97

.github/workflows/claude.yml

@@ -58,6 +58,6 @@ jobs:
           claude_env: |
             ANTHROPIC_AUTH_TOKEN: ${{ secrets.ANTHROPIC_AUTH_TOKEN }}
             ANTHROPIC_SMALL_FAST_MODEL: claude-3-5-haiku
-            CLAUDE_CODE_MAX_OUTPUT_TOKENS: 64000
+            CLAUDE_CODE_MAX_OUTPUT_TOKENS: 32000
         env:
-          ANTHROPIC_BASE_URL: ${{ secrets.ANTHROPIC_BASE_URL }}
+          ANTHROPIC_BASE_URL: ${{ secrets.ANTHROPIC_BASE_URL }}

api/sdk/src/main/java/com/ke/bella/openapi/protocol/IPriceInfo.java

@@ -6,6 +6,7 @@
 import com.ke.bella.openapi.protocol.realtime.RealTimePriceInfo;
 import com.ke.bella.openapi.protocol.completion.CompletionPriceInfo;
 import com.ke.bella.openapi.protocol.embedding.EmbeddingPriceInfo;
+import com.ke.bella.openapi.protocol.speaker.SpeakerEmbeddingPriceInfo;
 import com.ke.bella.openapi.protocol.tts.TtsPriceInfo;
 import com.ke.bella.openapi.protocol.images.ImagesPriceInfo;
 import lombok.AllArgsConstructor;
@@ -24,6 +25,7 @@ enum EndpointPriceInfoType {
         REALTIME_ASR("/v1/audio/asr/stream", RealTimePriceInfo.class),
         REALTIME("/v1/audio/realtime", RealTimePriceInfo.class),
         TRANSCRIPTION_ASR("/v1/audio/transcriptions", TranscriptionsAsrPriceInfo.class),
+        SPEAKER_EMBEDDING("/v1/audio/speaker/embedding", SpeakerEmbeddingPriceInfo.class),
         IMAGES("/v1/images/generations", ImagesPriceInfo.class),
         ;
 

api/sdk/src/main/java/com/ke/bella/openapi/protocol/document/parse/DocParseRequest.java

@@ -14,5 +14,7 @@ public class DocParseRequest implements UserRequest {
     private SourceFile file;
     private String user;
     private String model;
-    private String type; //task, block
+    private String type; //task, blocking
+    private String callbackUrl;
+    private int maxTimeoutMillis;
 }

api/sdk/src/main/java/com/ke/bella/openapi/protocol/realtime/RealTimeMessage.java

@@ -269,6 +269,20 @@ public static class Payload implements Serializable {
          * 变量配置
          */
         private Map<String, Object> variables;
+        
+        /**
+         * 热词字符串
+         * 用于语音识别热词配置
+         */
+        @JsonProperty("hot_words")
+        private String hotWords;
+        
+        /**
+         * 热词表ID
+         * 用于语音识别热词表配置
+         */
+        @JsonProperty("hot_words_table_id")
+        private String hotWordsTableId;
     }
 
     /**

api/sdk/src/main/java/com/ke/bella/openapi/protocol/speaker/SpeakerEmbeddingPriceInfo.java

@@ -0,0 +1,33 @@
+package com.ke.bella.openapi.protocol.speaker;
+
+import com.google.common.collect.ImmutableMap;
+import com.ke.bella.openapi.protocol.IPriceInfo;
+import lombok.Data;
+
+import java.io.Serializable;
+import java.math.BigDecimal;
+import java.util.Map;
+
+/**
+ * Price information for speaker embedding requests
+ * 基于音频时长计费
+ */
+@Data
+public class SpeakerEmbeddingPriceInfo implements IPriceInfo, Serializable {
+    private static final long serialVersionUID = 1L;
+    
+    /**
+     * 音频处理单价（元/小时）
+     */
+    private BigDecimal price;
+    
+    @Override
+    public String getUnit() {
+        return "元/小时";
+    }
+    
+    @Override
+    public Map<String, String> description() {
+        return ImmutableMap.of("price", "每小时价格（元）");
+    }
+}

api/sdk/src/main/java/com/ke/bella/openapi/protocol/speaker/SpeakerEmbeddingProperty.java

@@ -0,0 +1,12 @@
+package com.ke.bella.openapi.protocol.speaker;
+
+import lombok.Data;
+import org.apache.commons.lang3.StringUtils;
+
+/**
+ * Property class for speaker embedding providers
+ */
+@Data
+public class SpeakerEmbeddingProperty {
+    private String encodingType = StringUtils.EMPTY;
+}

api/sdk/src/main/java/com/ke/bella/openapi/protocol/speaker/SpeakerEmbeddingRequest.java

@@ -0,0 +1,37 @@
+package com.ke.bella.openapi.protocol.speaker;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.ke.bella.openapi.protocol.UserRequest;
+import lombok.Data;
+
+import java.io.Serializable;
+
+/**
+ * Speaker embedding request for generating voice embeddings from audio
+ */
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@Data
+public class SpeakerEmbeddingRequest implements UserRequest, Serializable {
+    private static final long serialVersionUID = 1L;
+    
+    private String url;
+    private String base64;
+    private String model;
+    private boolean normalize = true;
+    private String user = "";
+    @JsonProperty("sample_rate")
+    private int sampleRate = 16000;
+    @JsonProperty("task_id")
+    private String taskId = "";
+    
+    // Voice Activity Detection parameters
+    @JsonProperty("enable_vad")
+    private boolean enableVad = false;
+    @JsonProperty("vad_aggressiveness")
+    private int vadAggressiveness = 1;
+    @JsonProperty("min_speech_duration")
+    private double minSpeechDuration = 1.0;
+    @JsonProperty("max_silence_duration")
+    private double maxSilenceDuration = 0.5;
+}

api/sdk/src/main/java/com/ke/bella/openapi/protocol/speaker/SpeakerEmbeddingResponse.java

@@ -0,0 +1,49 @@
+package com.ke.bella.openapi.protocol.speaker;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.ke.bella.openapi.protocol.OpenapiResponse;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+
+import java.math.BigDecimal;
+import java.math.RoundingMode;
+import java.util.List;
+
+/**
+ * Speaker embedding response containing embeddings for audio segments
+ */
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@EqualsAndHashCode(callSuper = true)
+@Data
+public class SpeakerEmbeddingResponse extends OpenapiResponse {
+    private String task = "speaker_embedding";
+    @JsonProperty("task_id")
+    private String taskId = "";
+    private double duration = 0;
+    private List<Embedding> embeddings;
+    private int dimensions = 0;
+
+    @Data
+    public static class Embedding {
+        private int id = 0;
+        private double start = 0;
+        private double end = 0;
+        private double confidence = 0.0;
+        private List<Double> embedding;
+        
+        public void setStart(double start) {
+            this.start = roundToThreeDecimals(start);
+        }
+        
+        public void setEnd(double end) {
+            this.end = roundToThreeDecimals(end);
+        }
+        
+        private double roundToThreeDecimals(double value) {
+            return BigDecimal.valueOf(value)
+                    .setScale(3, RoundingMode.HALF_UP)
+                    .doubleValue();
+        }
+    }
+}

api/server/pom.xml

@@ -243,8 +243,19 @@
                     <groupId>org.springframework.boot</groupId>
                     <artifactId>spring-boot-starter-logging</artifactId>
                 </exclusion>
+                <!-- 排除不兼容的Redisson Spring Data适配器 -->
+                <exclusion>
+                    <groupId>org.redisson</groupId>
+                    <artifactId>redisson-spring-data-27</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
+        <!-- 显式添加兼容的Redisson Spring Data适配器 -->
+        <dependency>
+            <groupId>org.redisson</groupId>
+            <artifactId>redisson-spring-data-23</artifactId>
+            <version>3.17.7</version>
+        </dependency>
         <dependency>
             <groupId>com.lmax</groupId>
             <artifactId>disruptor</artifactId>
@@ -271,11 +282,6 @@
             <artifactId>cas-client-core</artifactId>
             <version>3.6.4</version>
         </dependency>
-        <dependency>
-            <groupId>org.springframework.data</groupId>
-            <artifactId>spring-data-redis</artifactId>
-            <version>2.2.0.RELEASE</version>
-        </dependency>
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>

api/server/src/main/java/com/ke/bella/openapi/endpoints/AudioController.java

@@ -22,6 +22,10 @@
 import com.ke.bella.openapi.protocol.log.EndpointLogger;
 import com.ke.bella.openapi.protocol.realtime.RealTimeAdaptor;
 import com.ke.bella.openapi.protocol.realtime.RealTimeHandler;
+import com.ke.bella.openapi.protocol.speaker.SpeakerEmbeddingAdaptor;
+import com.ke.bella.openapi.protocol.speaker.SpeakerEmbeddingProperty;
+import com.ke.bella.openapi.protocol.speaker.SpeakerEmbeddingRequest;
+import com.ke.bella.openapi.protocol.speaker.SpeakerEmbeddingResponse;
 import com.ke.bella.openapi.protocol.tts.TtsAdaptor;
 import com.ke.bella.openapi.protocol.tts.TtsProperty;
 import com.ke.bella.openapi.protocol.tts.TtsRequest;
@@ -269,4 +273,25 @@ public FlashAsrResponse flashAsr(@RequestHeader(value = "format", defaultValue =
         return adaptor.asr(request, url, property, processData);
     }
 
+    @PostMapping("/speaker/embedding")
+    @SuppressWarnings({ "rawtypes", "unchecked" })
+    public SpeakerEmbeddingResponse speakerEmbedding(@RequestBody SpeakerEmbeddingRequest request) {
+        String endpoint = EndpointContext.getRequest().getRequestURI();
+        String model = request.getModel();
+        EndpointContext.setEndpointData(endpoint, model, request);
+        EndpointProcessData processData = EndpointContext.getProcessData();
+        ChannelDB channel = router.route(endpoint, model, EndpointContext.getApikey(), processData.isMock());
+        EndpointContext.setEndpointData(channel);
+        if(!EndpointContext.getProcessData().isPrivate()) {
+            limiterManager.incrementConcurrentCount(EndpointContext.getProcessData().getAkCode(), model);
+        }
+        String protocol = processData.getProtocol();
+        String url = processData.getForwardUrl();
+        String channelInfo = channel.getChannelInfo();
+        SpeakerEmbeddingAdaptor adaptor = adaptorManager.getProtocolAdaptor(endpoint, protocol, SpeakerEmbeddingAdaptor.class);
+        SpeakerEmbeddingProperty property = (SpeakerEmbeddingProperty) JacksonUtils.deserialize(channelInfo, adaptor.getPropertyClass());
+        EndpointContext.setEncodingType(property.getEncodingType());
+        return adaptor.speakerEmbedding(request, url, property);
+    }
+
 }