Skip to content

Commit 7a2718b

Browse files
0xShug00xShug0
committed
Promote Fish DAC codec runtime
1 parent a4bf622 commit 7a2718b

14 files changed

Lines changed: 543 additions & 200 deletions

File tree

‎CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -468,6 +468,7 @@ add_library(engine_core OBJECT
468468
src/framework/modules/speaker_encoders/ecapa_tdnn_runtime.cpp
469469
src/framework/modules/speaker_encoders/titanet_runtime.cpp
470470
src/framework/codecs/fsq_audio_codec_runtime.cpp
471+
src/framework/codecs/fish_dac_codec_runtime.cpp
471472
src/framework/codecs/mel_latent_vae44k_runtime.cpp
472473
src/framework/codecs/mimi_codec_runtime.cpp
473474
src/framework/codecs/neural_audio.cpp
@@ -903,7 +904,6 @@ audiocpp_add_model(fish_audio
903904
SOURCES
904905
src/models/fish_audio/ar.cpp
905906
src/models/fish_audio/assets.cpp
906-
src/models/fish_audio/codec.cpp
907907
src/models/fish_audio/generator.cpp
908908
src/models/fish_audio/loader.cpp
909909
src/models/fish_audio/prompt_builder.cpp

‎include/engine/framework/codecs/fish_dac_codec_runtime.h‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
#pragma once
2+
3+
#include "engine/framework/assets/tensor_source.h"
4+
#include "engine/framework/core/backend.h"
5+
#include "engine/framework/runtime/session.h"
6+
7+
#include <cstddef>
8+
#include <cstdint>
9+
#include <memory>
10+
#include <string>
11+
#include <vector>
12+
13+
namespace engine::codecs {
14+
15+
struct FishDacCodes {
16+
std::vector<int32_t> codes;
17+
int64_t codebooks = 0;
18+
int64_t frames = 0;
19+
};
20+
21+
struct FishDacLatents {
22+
std::vector<float> values;
23+
int64_t frames = 0;
24+
int64_t channels = 0;
25+
};
26+
27+
struct FishDacCodecConfig {
28+
int sample_rate = 44100;
29+
int64_t semantic_codebook_size = 4096;
30+
int64_t residual_codebook_size = 1024;
31+
int64_t quantizer_codebooks = 9;
32+
int64_t total_codebooks = 10;
33+
int64_t codebook_dim = 8;
34+
int64_t latent_dim = 1024;
35+
int64_t frame_length = 2048;
36+
};
37+
38+
struct FishDacCodecWeightBinding {
39+
std::string encoder_prefix = "encoder";
40+
std::string quantizer_prefix = "quantizer";
41+
std::string decoder_prefix = "decoder";
42+
};
43+
44+
struct FishDacCodecRuntimeOptions {
45+
size_t graph_arena_bytes = 512ull * 1024ull * 1024ull;
46+
size_t weight_context_bytes = 1024ull * 1024ull * 1024ull;
47+
assets::TensorStorageType matmul_weight_storage_type = assets::TensorStorageType::Native;
48+
assets::TensorStorageType conv_weight_storage_type = assets::TensorStorageType::Native;
49+
};
50+
51+
class FishDacCodecComponent {
52+
public:
53+
static std::shared_ptr<const FishDacCodecComponent> load_from_tensor_source(
54+
std::shared_ptr<const assets::TensorSource> source,
55+
FishDacCodecConfig config,
56+
FishDacCodecWeightBinding binding,
57+
ggml_backend_t backend,
58+
core::BackendType backend_type,
59+
FishDacCodecRuntimeOptions options);
60+
61+
~FishDacCodecComponent();
62+
63+
const FishDacCodecConfig & config() const noexcept;
64+
65+
private:
66+
struct Impl;
67+
explicit FishDacCodecComponent(std::shared_ptr<const Impl> impl);
68+
69+
std::shared_ptr<const Impl> impl_;
70+
71+
friend class FishDacCodecRuntime;
72+
};
73+
74+
class FishDacCodecRuntime {
75+
public:
76+
FishDacCodecRuntime(
77+
std::shared_ptr<const FishDacCodecComponent> component,
78+
core::BackendConfig backend,
79+
int threads,
80+
size_t graph_arena_bytes);
81+
~FishDacCodecRuntime();
82+
83+
FishDacCodecRuntime(const FishDacCodecRuntime &) = delete;
84+
FishDacCodecRuntime & operator=(const FishDacCodecRuntime &) = delete;
85+
86+
FishDacCodes encode_codes(const runtime::AudioBuffer & audio);
87+
FishDacLatents encode_latents(const runtime::AudioBuffer & audio);
88+
runtime::AudioBuffer decode_codes(const FishDacCodes & codes);
89+
runtime::AudioBuffer decode_latents(const FishDacLatents & latents);
90+
runtime::AudioBuffer decode_latents(const std::vector<float> & values, int64_t frames);
91+
void release_encode_graph();
92+
void release_runtime_graphs();
93+
94+
private:
95+
class Impl;
96+
std::unique_ptr<Impl> impl_;
97+
};
98+
99+
} // namespace engine::codecs

‎include/engine/models/fish_audio/ar.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ class FishAudioARRuntime {
2020
assets::TensorStorageType weight_storage_type);
2121
~FishAudioARRuntime();
2222

23-
FishAudioCodes generate(const FishAudioPrompt & prompt, const FishAudioGenerationOptions & options);
23+
engine::codecs::FishDacCodes generate(const FishAudioPrompt & prompt, const FishAudioGenerationOptions & options);
2424
void release_runtime_graphs();
2525

2626
private:

‎include/engine/models/fish_audio/codec.h‎

Lines changed: 0 additions & 34 deletions
This file was deleted.

‎include/engine/models/fish_audio/generator.h‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#pragma once
22

3+
#include "engine/framework/codecs/fish_dac_codec_runtime.h"
34
#include "engine/models/fish_audio/ar.h"
4-
#include "engine/models/fish_audio/codec.h"
55
#include "engine/models/fish_audio/prompt_builder.h"
66
#include "engine/models/fish_audio/tokenizer_text.h"
77

@@ -12,21 +12,21 @@ namespace engine::models::fish_audio {
1212

1313
struct FishAudioGenerationResult {
1414
runtime::AudioBuffer audio;
15-
FishAudioCodes codes;
15+
engine::codecs::FishDacCodes codes;
1616
};
1717

1818
class FishAudioGenerator {
1919
public:
2020
FishAudioGenerator(
2121
std::shared_ptr<const FishAudioAssets> assets,
2222
std::unique_ptr<FishAudioARRuntime> ar,
23-
std::unique_ptr<FishAudioCodecRuntime> codec);
23+
std::unique_ptr<engine::codecs::FishDacCodecRuntime> codec);
2424
~FishAudioGenerator();
2525

26-
FishAudioCodes encode_reference(const runtime::AudioBuffer & audio);
26+
engine::codecs::FishDacCodes encode_reference(const runtime::AudioBuffer & audio);
2727
FishAudioGenerationResult generate(
2828
const FishAudioRequest & request,
29-
const std::vector<FishAudioCodes> & reference_codes,
29+
const std::vector<engine::codecs::FishDacCodes> & reference_codes,
3030
const std::optional<FishAudioConversationTurn> & previous_turn,
3131
bool mem_saver);
3232

@@ -35,7 +35,7 @@ class FishAudioGenerator {
3535
FishAudioTextTokenizer tokenizer_;
3636
FishAudioPromptBuilder prompt_builder_;
3737
std::unique_ptr<FishAudioARRuntime> ar_;
38-
std::unique_ptr<FishAudioCodecRuntime> codec_;
38+
std::unique_ptr<engine::codecs::FishDacCodecRuntime> codec_;
3939
};
4040

4141
} // namespace engine::models::fish_audio

‎include/engine/models/fish_audio/prompt_builder.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ class FishAudioPromptBuilder {
1111

1212
FishAudioPrompt build(
1313
const FishAudioRequest & request,
14-
const std::vector<FishAudioCodes> & reference_codes,
14+
const std::vector<engine::codecs::FishDacCodes> & reference_codes,
1515
const std::optional<FishAudioConversationTurn> & previous_turn) const;
1616

1717
private:

‎include/engine/models/fish_audio/session.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,11 +41,11 @@ class FishAudioSession final : public runtime::RuntimeSessionBase, public runtim
4141
};
4242

4343
struct ReferenceCacheEntry {
44-
FishAudioCodes codes;
44+
engine::codecs::FishDacCodes codes;
4545
};
4646

4747
FishAudioRequest make_request(const runtime::TaskRequest & request) const;
48-
const FishAudioCodes & resolve_reference_codes(const FishAudioReference & reference);
48+
const engine::codecs::FishDacCodes & resolve_reference_codes(const FishAudioReference & reference);
4949

5050
runtime::TaskSpec task_;
5151
std::shared_ptr<const FishAudioAssets> assets_;

‎include/engine/models/fish_audio/types.h‎

Lines changed: 3 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
#pragma once
22

3+
#include "engine/framework/codecs/fish_dac_codec_runtime.h"
34
#include "engine/framework/runtime/session.h"
45

56
#include <cstdint>
@@ -30,15 +31,9 @@ struct FishAudioRequest {
3031
FishAudioGenerationOptions generation;
3132
};
3233

33-
struct FishAudioCodes {
34-
std::vector<int32_t> codes;
35-
int64_t codebooks = 0;
36-
int64_t frames = 0;
37-
};
38-
3934
struct FishAudioConversationTurn {
4035
std::string text;
41-
FishAudioCodes codes;
36+
engine::codecs::FishDacCodes codes;
4237
};
4338

4439
struct FishAudioPrompt {
@@ -79,17 +74,6 @@ struct FishAudioFastConfig {
7974
bool attention_qk_norm = false;
8075
};
8176

82-
struct FishAudioCodecConfig {
83-
int sample_rate = 44100;
84-
int64_t semantic_codebook_size = 4096;
85-
int64_t residual_codebook_size = 1024;
86-
int64_t quantizer_codebooks = 9;
87-
int64_t total_codebooks = 10;
88-
int64_t codebook_dim = 8;
89-
int64_t latent_dim = 1024;
90-
int64_t frame_length = 2048;
91-
};
92-
9377
struct FishAudioConfig {
9478
std::string model_type;
9579
std::string torch_dtype;
@@ -99,7 +83,7 @@ struct FishAudioConfig {
9983
bool norm_fastlayer_input = false;
10084
FishAudioTextConfig text;
10185
FishAudioFastConfig fast;
102-
FishAudioCodecConfig codec;
86+
engine::codecs::FishDacCodecConfig codec;
10387
};
10488

10589
} // namespace engine::models::fish_audio

‎model_specs/fish_audio.json‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -139,10 +139,10 @@
139139
{
140140
"name": "ar_weight_context_mb",
141141
"type": "int",
142-
"description": "AR weight context size in MiB; default 512.",
142+
"description": "AR weight context size in MiB; default 32.",
143143
"required": false,
144144
"min": 0,
145-
"default": 512
145+
"default": 32
146146
},
147147
{
148148
"name": "codec_graph_arena_mb",
@@ -155,10 +155,10 @@
155155
{
156156
"name": "codec_weight_context_mb",
157157
"type": "int",
158-
"description": "Codec weight context size in MiB; default 512.",
158+
"description": "Codec weight context size in MiB; default 32.",
159159
"required": false,
160160
"min": 0,
161-
"default": 512
161+
"default": 32
162162
}
163163
],
164164
"load": []

0 commit comments

Comments
 (0)