feat(model/omni):support session.audio.input/output

Author: songguocola

dashscope/audio/qwen_omni/__init__.py

@@ -3,6 +3,9 @@
 
 from .omni_realtime import (
     AudioFormat,
+    AudioFormatConfig,
+    AudioFormatType,
+    AudioSampleRate,
     MultiModality,
     OmniRealtimeCallback,
     OmniRealtimeConversation,
@@ -11,6 +14,9 @@
 __all__ = [
     "OmniRealtimeCallback",
     "AudioFormat",
+    "AudioFormatConfig",
+    "AudioFormatType",
+    "AudioSampleRate",
     "MultiModality",
     "OmniRealtimeConversation",
 ]

dashscope/audio/qwen_omni/omni_realtime.py

@@ -87,6 +87,92 @@ def __str__(self):
         return f"{self.format.upper()} with {self.sample_rate}Hz sample rate, {self.channels} channel, {self.bit_rate} bit rate:  {self.format_str}"  # noqa: E501  # pylint: disable=line-too-long
 
 
+class AudioFormatType(Enum):
+    """
+    Preset audio format types for omni realtime input/output audio.
+
+    These are provided for convenient reference only. ``AudioFormatConfig``
+    also accepts any raw string value (e.g. ``"mp3"``) without validation,
+    so new formats can be used without upgrading the SDK.
+    """
+
+    PCM = "pcm"
+    WAV = "wav"
+
+    def __str__(self):
+        return self.value
+
+
+@unique
+class AudioSampleRate(Enum):
+    """
+    Preset audio sample rates(Hz) for omni realtime input/output audio.
+
+    These are provided for convenient reference only. ``AudioFormatConfig``
+    also accepts any raw int value without validation.
+    """
+
+    SAMPLE_RATE_8K = 8000
+    SAMPLE_RATE_16K = 16000
+    SAMPLE_RATE_24K = 24000
+    SAMPLE_RATE_48K = 48000
+
+    def __int__(self):
+        return self.value
+
+
+@dataclass
+class AudioFormatConfig:
+    """
+    Audio format config for input(uplink) / output(downlink) audio in a
+    omni realtime session.
+
+    No strict validation is performed: the preset enums are only for
+    convenient reference, while any raw value is also accepted so that new
+    formats or sample rates can be used without upgrading the SDK.
+
+    Parameters
+    ----------
+    type: str
+        audio format type. Presets ``AudioFormatType.PCM`` / ``.WAV`` are
+        provided for convenience; accepts an ``AudioFormatType`` enum or any
+        raw string such as ``"pcm"`` / ``"wav"`` / ``"mp3"``.
+    sample_rate: int
+        audio sample rate in Hz. Presets ``AudioSampleRate`` (8000 / 16000 /
+        24000 / 48000) are provided for convenience; accepts an
+        ``AudioSampleRate`` enum or any raw int value.
+    extra_params: Dict[str, Any]
+        free extension parameters that will be merged into the ``format``
+        dict. Reserved for future parameters such as speech rate, e.g.
+        ``{"speech_rate": 1.2}``.
+    """
+
+    type: Any = AudioFormatType.PCM.value
+    sample_rate: Any = AudioSampleRate.SAMPLE_RATE_16K.value
+    extra_params: Dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> Dict[str, Any]:
+        """
+        convert to the ``format`` dict used in the session.update request.
+
+        Preset enums are unwrapped to their underlying value; any raw value
+        is passed through as-is without validation.
+        """
+        format_type = self.type
+        if isinstance(format_type, AudioFormatType):
+            format_type = format_type.value
+        sample_rate = self.sample_rate
+        if isinstance(sample_rate, AudioSampleRate):
+            sample_rate = sample_rate.value
+        result: Dict[str, Any] = {
+            "type": format_type,
+            "sample_rate": sample_rate,
+        }
+        if self.extra_params:
+            result.update(self.extra_params)
+        return result
+
+
 class MultiModality(Enum):
     """
     MultiModality
@@ -232,6 +318,35 @@ def create_item(self, item: dict):
             enable_log=True,
         )
 
+    def _apply_audio_format(
+        self,
+        input_audio_format: AudioFormat,
+        output_audio_format: AudioFormat,
+        input_audio_config: AudioFormatConfig,
+        output_audio_config: AudioFormatConfig,
+    ) -> None:
+        """
+        apply audio format config into ``self.config``.
+
+        Prefer the new-style ``session.audio.{input,output}.format`` structure
+        when ``input_audio_config`` / ``output_audio_config`` is provided,
+        otherwise keep the old-style top-level fields for backward
+        compatibility.
+        """
+        if input_audio_config is None and output_audio_config is None:
+            # old-style: keep backward compatibility
+            self.config["input_audio_format"] = input_audio_format.format_str
+            self.config["output_audio_format"] = output_audio_format.format_str
+            return
+        # new-style: separate audio format type and sample rate under
+        # session.audio.{input,output}.format
+        audio: Dict[str, Any] = {}
+        if input_audio_config is not None:
+            audio["input"] = {"format": input_audio_config.to_dict()}
+        if output_audio_config is not None:
+            audio["output"] = {"format": output_audio_config.to_dict()}
+        self.config["audio"] = audio
+
     def update_session(
         self,
         output_modalities: List[MultiModality],
@@ -248,6 +363,8 @@ def update_session(
         turn_detection_param: dict = None,
         translation_params: TranslationParams = None,
         transcription_params: TranscriptionParams = None,
+        input_audio_config: AudioFormatConfig = None,
+        output_audio_config: AudioFormatConfig = None,
         **kwargs,
     ) -> None:
         """
@@ -260,9 +377,13 @@ def update_session(
         voice: str
             voice to be used in session
         input_audio_format: AudioFormat
-            input audio format
+            input audio format. Deprecated, kept for backward compatibility.
+            Prefer ``input_audio_config`` which supports separate audio
+            format type and sample rate.
         output_audio_format: AudioFormat
-            output audio format
+            output audio format. Deprecated, kept for backward compatibility.
+            Prefer ``output_audio_config`` which supports separate audio
+            format type and sample rate.
         enable_turn_detection: bool
             enable turn detection
         turn_detection_threshold: float
@@ -278,13 +399,27 @@ def update_session(
             transcription params, include language, sample_rate, input_audio_format, corpus.  # noqa: E501  # pylint: disable=line-too-long
             Only effective with qwen3-asr-flash-realtime model or
             further models. Do not set this parameter for other models.
+        input_audio_config: AudioFormatConfig
+            input(uplink) audio format config. Supports separate format type
+            (pcm/wav) and sample rate (8000/16000/24000/48000), as well as
+            free extension parameters via ``extra_params``. When provided,
+            the request emits the ``session.audio.input.format`` structure.
+        output_audio_config: AudioFormatConfig
+            output(downlink) audio format config. Supports separate format
+            type (pcm/wav) and sample rate (8000/16000/24000/48000), as well
+            as free extension parameters via ``extra_params``. When provided,
+            the request emits the ``session.audio.output.format`` structure.
         """
         self.config = {
             "modalities": [m.value for m in output_modalities],
             "voice": voice,
-            "input_audio_format": input_audio_format.format_str,
-            "output_audio_format": output_audio_format.format_str,
         }
+        self._apply_audio_format(
+            input_audio_format,
+            output_audio_format,
+            input_audio_config,
+            output_audio_config,
+        )
         if enable_input_audio_transcription:
             self.config["input_audio_transcription"] = {
                 "model": input_audio_transcription_model,