feat(diffusers): add AudioLDM2 generation

Expose diffusers audio pipelines through the existing sound-generation RPC. AudioLDM2 can now return PCM WAV output from the model gallery without a separate backend.

Assisted-by: Codex:gpt-5

Author: localai-org-maint-bot

backend/index.yaml

@@ -1194,6 +1194,7 @@
   tags:
     - image-generation
     - video-generation
+    - sound-generation
     - diffusion-models
   license: apache-2.0
   alias: "diffusers"

backend/python/diffusers/audio_utils.py

@@ -0,0 +1,24 @@
+import array
+import sys
+import wave
+
+
+def write_pcm_wav(destination, samples, sampling_rate):
+    """Write normalized floating-point audio samples as mono 16-bit PCM."""
+    pcm = array.array(
+        "h",
+        (
+            max(-32768, min(32767, round(float(sample) * 32768)))
+            for sample in samples
+        ),
+    )
+    if pcm.itemsize != 2:
+        raise RuntimeError("16-bit PCM requires two-byte signed integers")
+    if sys.byteorder != "little":
+        pcm.byteswap()
+
+    with wave.open(destination, "wb") as output:
+        output.setnchannels(1)
+        output.setsampwidth(2)
+        output.setframerate(sampling_rate)
+        output.writeframes(pcm.tobytes())

backend/python/diffusers/backend.py

@@ -26,6 +26,7 @@
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'common'))
 from grpc_auth import get_auth_interceptors
 from model_utils import resolve_model_reference
+from audio_utils import write_pcm_wav
 
 
 # Import dynamic loader for pipeline discovery
@@ -883,6 +884,49 @@ def GenerateImage(self, request, context):
 
         return backend_pb2.Result(message="Media generated", success=True)
 
+    def SoundGeneration(self, request, context):
+        if not request.dst:
+            return backend_pb2.Result(success=False, message="request.dst is required")
+
+        prompt = request.text or request.caption
+        if not prompt:
+            return backend_pb2.Result(success=False, message="request.text is required")
+
+        try:
+            generation_options = dict(self.options)
+            if "num_inference_steps" in generation_options:
+                generation_options["num_inference_steps"] = int(
+                    generation_options["num_inference_steps"]
+                )
+            generation_options["prompt"] = prompt
+            if request.HasField("duration"):
+                generation_options["audio_length_in_s"] = request.duration
+            if request.HasField("temperature"):
+                generation_options["guidance_scale"] = request.temperature
+
+            generated = self.pipe(**generation_options)
+            if not hasattr(generated, "audios") or len(generated.audios) == 0:
+                return backend_pb2.Result(
+                    success=False,
+                    message="The diffusers pipeline returned no audio",
+                )
+
+            samples = generated.audios[0]
+            if hasattr(samples, "reshape"):
+                samples = samples.reshape(-1)
+            if hasattr(samples, "tolist"):
+                samples = samples.tolist()
+
+            sampling_rate = getattr(
+                getattr(getattr(self.pipe, "vae", None), "config", None),
+                "sampling_rate",
+                16000,
+            )
+            write_pcm_wav(request.dst, samples, sampling_rate)
+            return backend_pb2.Result(success=True, message="Sound generated successfully")
+        except Exception as err:
+            return backend_pb2.Result(success=False, message=f"SoundGeneration error: {err}")
+
     def UpscaleImage(self, request, context):
         try:
             if not request.src:

backend/python/diffusers/test.py

@@ -4,6 +4,9 @@
 import unittest
 import subprocess
 import time
+import os
+import tempfile
+import wave
 from unittest.mock import patch, MagicMock
 
 # Import dynamic loader for testing (these don't need gRPC)
@@ -373,3 +376,64 @@ def test_options_merged_into_pipeline_kwargs(self):
         finally:
             os.unlink(src_file.name)
             os.unlink(dst_file.name)
+
+
+class TestWritePcmWav(unittest.TestCase):
+    def test_writes_clipped_float_samples_as_mono_pcm(self):
+        from audio_utils import write_pcm_wav
+
+        destination = tempfile.NamedTemporaryFile(suffix=".wav", delete=False)
+        destination.close()
+
+        try:
+            write_pcm_wav(destination.name, [0.0, 0.5, -0.5, 2.0], 16000)
+
+            with wave.open(destination.name, "rb") as generated:
+                self.assertEqual(generated.getframerate(), 16000)
+                self.assertEqual(generated.getnchannels(), 1)
+                self.assertEqual(generated.getsampwidth(), 2)
+                self.assertEqual(generated.getnframes(), 4)
+                self.assertEqual(
+                    generated.readframes(4),
+                    b"\x00\x00\x00@\x00\xc0\xff\x7f",
+                )
+        finally:
+            os.unlink(destination.name)
+
+
+@unittest.skipUnless(GRPC_AVAILABLE, "gRPC modules not available")
+class TestSoundGeneration(unittest.TestCase):
+    def test_maps_request_options_and_writes_pipeline_audio(self):
+        from backend import BackendServicer
+
+        service = BackendServicer.__new__(BackendServicer)
+        service.options = {"num_inference_steps": 200.0}
+        service.pipe = MagicMock()
+        service.pipe.return_value.audios = [[0.0, 0.5, -0.5]]
+        service.pipe.vae.config.sampling_rate = 16000
+
+        destination = tempfile.NamedTemporaryFile(suffix=".wav", delete=False)
+        destination.close()
+
+        try:
+            request = backend_pb2.SoundGenerationRequest(
+                text="ocean waves",
+                dst=destination.name,
+                duration=2.5,
+                temperature=0,
+            )
+
+            result = service.SoundGeneration(request, context=None)
+
+            self.assertTrue(result.success, result.message)
+            service.pipe.assert_called_once_with(
+                num_inference_steps=200,
+                prompt="ocean waves",
+                audio_length_in_s=2.5,
+                guidance_scale=0,
+            )
+            with wave.open(destination.name, "rb") as generated:
+                self.assertEqual(generated.getframerate(), 16000)
+                self.assertEqual(generated.getnframes(), 3)
+        finally:
+            os.unlink(destination.name)

core/config/backend_capabilities.go

@@ -369,10 +369,10 @@ var BackendCapabilities = map[string]BackendCapability{
 
 	// --- Image/video generation backends ---
 	"diffusers": {
-		GRPCMethods:      []GRPCMethod{MethodGenerateImage, MethodUpscaleImage, MethodGenerateVideo},
-		PossibleUsecases: []string{UsecaseImage, UsecaseVideo},
+		GRPCMethods:      []GRPCMethod{MethodGenerateImage, MethodUpscaleImage, MethodGenerateVideo, MethodSoundGeneration},
+		PossibleUsecases: []string{UsecaseImage, UsecaseVideo, UsecaseSoundGeneration},
 		DefaultUsecases:  []string{UsecaseImage},
-		Description:      "HuggingFace diffusers — Stable Diffusion, Flux, video generation",
+		Description:      "HuggingFace diffusers — image, video, and sound generation",
 	},
 	"longcat-video": {
 		GRPCMethods:      []GRPCMethod{MethodGenerateVideo},

core/config/backend_capabilities_test.go

@@ -57,6 +57,12 @@ var _ = Describe("BackendCapabilities", func() {
 })
 
 var _ = Describe("GetBackendCapability", func() {
+	It("advertises diffusers sound generation", func() {
+		capability := GetBackendCapability("diffusers")
+		Expect(capability.GRPCMethods).To(ContainElement(MethodSoundGeneration))
+		Expect(capability.PossibleUsecases).To(ContainElement(UsecaseSoundGeneration))
+	})
+
 	It("returns the capability for a known backend", func() {
 		cap := GetBackendCapability("llama-cpp")
 		Expect(cap).NotTo(BeNil())

docs/content/features/text-to-audio.md

@@ -302,6 +302,31 @@ The `/v1/sound-generation` endpoint is compatible with the [ElevenLabs sound gen
 
 Error responses: `400` for a missing or invalid model or request parameters, and `500` for a backend error during sound generation.
 
+### AudioLDM 2
+
+[AudioLDM 2](https://github.com/haoheliu/AudioLDM2) generates sound effects,
+music, and speech from a text description. Install the gallery model:
+
+```bash
+local-ai models install audioldm2
+```
+
+Generate a WAV file through the sound-generation endpoint:
+
+```bash
+curl http://localhost:8080/v1/sound-generation \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model_id": "audioldm2",
+    "text": "Waves breaking on a rocky beach during a distant thunderstorm",
+    "duration_seconds": 10
+  }' --output storm.wav
+```
+
+AudioLDM 2 uses the `AudioLDM2Pipeline` from the diffusers backend. The
+`duration_seconds` field maps to the pipeline's `audio_length_in_s` option, and
+`prompt_influence` maps to `guidance_scale`.
+
 #### Configuration
 
 You can configure ACE-Step models with various options:

gallery/audioldm2.yaml

@@ -0,0 +1,15 @@
+---
+name: "audioldm2"
+
+config_file: |
+  backend: diffusers
+  known_usecases:
+    - sound_generation
+  parameters:
+    model: cvssp/audioldm2
+  diffusers:
+    pipeline_type: AudioLDM2Pipeline
+    cuda: true
+  options:
+    - num_inference_steps:200
+    - torch_dtype:fp16

gallery/index.yaml

@@ -1,4 +1,20 @@
 ---
+- name: audioldm2
+  url: github:mudler/LocalAI/gallery/audioldm2.yaml@master
+  urls:
+    - https://huggingface.co/cvssp/audioldm2
+    - https://github.com/haoheliu/AudioLDM2
+  description: |
+    AudioLDM 2 generates sound effects, music, and speech from natural-language
+    descriptions through the diffusers backend and LocalAI sound-generation API.
+  license: cc-by-nc-sa-4.0
+  tags:
+    - audio
+    - sound-generation
+    - text-to-audio
+    - diffusers
+    - gpu
+  last_checked: "2026-08-13"
 - &twil-lm3
   name: "twil-lm3-q4"
   variants: