|
| 1 | +#pragma once |
| 2 | + |
| 3 | +#include "engine/framework/assets/tensor_source.h" |
| 4 | +#include "engine/framework/core/execution_context.h" |
| 5 | + |
| 6 | +#include <cstddef> |
| 7 | +#include <cstdint> |
| 8 | +#include <memory> |
| 9 | +#include <vector> |
| 10 | + |
| 11 | +namespace engine::codecs { |
| 12 | + |
| 13 | +struct MossAudioTokenizerTransformerStage { |
| 14 | + int64_t input_dimension = 0; |
| 15 | + int64_t output_dimension = 0; |
| 16 | + int64_t model_dimension = 0; |
| 17 | + int64_t num_heads = 0; |
| 18 | + int64_t num_layers = 0; |
| 19 | + int64_t feedforward_dimension = 0; |
| 20 | + int64_t context_window = 0; |
| 21 | + int64_t patch_size = 0; |
| 22 | +}; |
| 23 | + |
| 24 | +struct MossAudioTokenizerQuantizerConfig { |
| 25 | + int64_t codebook_size = 1024; |
| 26 | + int64_t codebook_dim = 8; |
| 27 | + int64_t rvq_dim = 512; |
| 28 | + int64_t code_dim = 768; |
| 29 | + int64_t num_quantizers = 12; |
| 30 | +}; |
| 31 | + |
| 32 | +struct MossAudioTokenizerConfig { |
| 33 | + int64_t sampling_rate = 48000; |
| 34 | + int64_t samples_per_frame = 3840; |
| 35 | + int64_t channels = 2; |
| 36 | + MossAudioTokenizerQuantizerConfig quantizer; |
| 37 | + std::vector<MossAudioTokenizerTransformerStage> encoder_stages; |
| 38 | + std::vector<MossAudioTokenizerTransformerStage> decoder_stages; |
| 39 | + int64_t encoder_final_patch = 1; |
| 40 | + int64_t decoder_initial_patch = 1; |
| 41 | + int64_t encoder_module_start = 1; |
| 42 | + int64_t encoder_module_stride = 2; |
| 43 | + int64_t decoder_module_start = 0; |
| 44 | + int64_t decoder_module_stride = 2; |
| 45 | +}; |
| 46 | + |
| 47 | +MossAudioTokenizerConfig moss_audio_tokenizer_v1_config(); |
| 48 | +MossAudioTokenizerConfig moss_audio_tokenizer_v2_config(); |
| 49 | +MossAudioTokenizerConfig moss_audio_tokenizer_nano_config(); |
| 50 | + |
| 51 | +struct MossAudioTokenizerAudio { |
| 52 | + int64_t sampling_rate = 0; |
| 53 | + std::vector<std::vector<float>> channels; |
| 54 | +}; |
| 55 | + |
| 56 | +struct MossAudioTokenizerCodes { |
| 57 | + int64_t frames = 0; |
| 58 | + std::vector<std::vector<int32_t>> codebooks; |
| 59 | +}; |
| 60 | + |
| 61 | +struct MossTokenRows { |
| 62 | + std::vector<int32_t> text_tokens; |
| 63 | + std::vector<int32_t> audio_codes; |
| 64 | +}; |
| 65 | + |
| 66 | +class MossTokenRowBuilder { |
| 67 | +public: |
| 68 | + MossTokenRowBuilder(int64_t num_codebooks, int32_t audio_pad_token_id); |
| 69 | + |
| 70 | + void push_text_token(int32_t token_id); |
| 71 | + void push_text_tokens(const std::vector<int32_t> & token_ids); |
| 72 | + void push_audio_row(int32_t text_slot_token_id, const int32_t * codes, int64_t num_codebooks); |
| 73 | + void push_audio_row(int32_t text_slot_token_id, const std::vector<std::vector<int32_t>> & codes, int64_t frame); |
| 74 | + MossTokenRows finish(); |
| 75 | + |
| 76 | +private: |
| 77 | + int64_t num_codebooks_ = 0; |
| 78 | + int32_t audio_pad_token_id_ = 0; |
| 79 | + MossTokenRows rows_; |
| 80 | +}; |
| 81 | + |
| 82 | +struct MossAudioTokenizerCodecRuntimeOptions { |
| 83 | + size_t weight_context_bytes = 256ull * 1024ull * 1024ull; |
| 84 | + size_t encoder_graph_arena_bytes = 2048ull * 1024ull * 1024ull; |
| 85 | + size_t decoder_graph_arena_bytes = 1536ull * 1024ull * 1024ull; |
| 86 | + bool separate_encoder_context = false; |
| 87 | +}; |
| 88 | + |
| 89 | +class MossAudioTokenizerCodecRuntime { |
| 90 | +public: |
| 91 | + MossAudioTokenizerCodecRuntime( |
| 92 | + std::shared_ptr<const assets::TensorSource> source, |
| 93 | + core::ExecutionContext & execution_context, |
| 94 | + int64_t num_quantizers, |
| 95 | + MossAudioTokenizerCodecRuntimeOptions options, |
| 96 | + MossAudioTokenizerConfig config = moss_audio_tokenizer_v2_config()); |
| 97 | + ~MossAudioTokenizerCodecRuntime(); |
| 98 | + |
| 99 | + MossAudioTokenizerCodecRuntime(const MossAudioTokenizerCodecRuntime &) = delete; |
| 100 | + MossAudioTokenizerCodecRuntime & operator=(const MossAudioTokenizerCodecRuntime &) = delete; |
| 101 | + |
| 102 | + int64_t sampling_rate() const noexcept; |
| 103 | + void prepare_encoder(); |
| 104 | + void prepare_decoder(); |
| 105 | + MossAudioTokenizerCodes encode(const MossAudioTokenizerAudio & audio); |
| 106 | + MossAudioTokenizerAudio decode(const MossAudioTokenizerCodes & codes); |
| 107 | + void release_runtime_graphs(); |
| 108 | + |
| 109 | +private: |
| 110 | + struct Impl; |
| 111 | + std::unique_ptr<Impl> impl_; |
| 112 | +}; |
| 113 | + |
| 114 | +} // namespace engine::codecs |
0 commit comments