Skip to content

Commit bc7759f

Browse files
committed
Add optional server frontend pipeline
1 parent 89a0e98 commit bc7759f

12 files changed

Lines changed: 97137 additions & 4 deletions

File tree

CMakeLists.txt

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,15 @@ option(ENGINE_ENABLE_OPENMP "Build host code with OpenMP support" ON)
9898
option(AUDIOCPP_BUILD_NATIVE_MODEL_MANAGER
9999
"Build native model-manager tools and server download/install support"
100100
OFF)
101+
option(AUDIOCPP_BUILD_SERVER_FRONTENDS
102+
"Build optional in-process audiocpp_server frontend adapters"
103+
OFF)
104+
set(AUDIOCPP_SERVER_FRONTEND_MODULES ""
105+
CACHE STRING "Semicolon-separated optional audiocpp_server frontend modules to build")
106+
if (NOT AUDIOCPP_BUILD_SERVER_FRONTENDS AND AUDIOCPP_SERVER_FRONTEND_MODULES)
107+
message(FATAL_ERROR
108+
"AUDIOCPP_SERVER_FRONTEND_MODULES requires AUDIOCPP_BUILD_SERVER_FRONTENDS=ON")
109+
endif()
101110
option(AUDIOCPP_USE_SYSTEM_OPENSSL
102111
"Use system OpenSSL for native model management instead of bundled BoringSSL"
103112
OFF)
@@ -1777,6 +1786,7 @@ add_executable(audiocpp_server
17771786
app/server/main.cpp
17781787
app/server/base64.cpp
17791788
app/server/config.cpp
1789+
app/server/frontend.cpp
17801790
app/server/http.cpp
17811791
app/server/model_memory.cpp
17821792
app/server/multipart.cpp
@@ -1789,6 +1799,75 @@ add_executable(audiocpp_server
17891799
)
17901800

17911801
target_link_libraries(audiocpp_server PRIVATE engine_runtime ggml)
1802+
set(AUDIOCPP_SERVER_FRONTEND_DECLARATIONS "")
1803+
set(AUDIOCPP_SERVER_FRONTEND_REGISTRATIONS "")
1804+
if (AUDIOCPP_BUILD_SERVER_FRONTENDS)
1805+
set(AUDIOCPP_SERVER_FRONTEND_SOURCES "")
1806+
set(AUDIOCPP_SERVER_FRONTEND_INCLUDE_DIRS "")
1807+
set(AUDIOCPP_SERVER_FRONTEND_LIBRARIES "")
1808+
foreach(AUDIOCPP_SERVER_FRONTEND_MODULE IN LISTS AUDIOCPP_SERVER_FRONTEND_MODULES)
1809+
if (AUDIOCPP_SERVER_FRONTEND_MODULE STREQUAL "audio_decode")
1810+
string(APPEND AUDIOCPP_SERVER_FRONTEND_DECLARATIONS
1811+
"void register_audio_decode_module(ServerFrontendRegistry & registry);\n")
1812+
string(APPEND AUDIOCPP_SERVER_FRONTEND_REGISTRATIONS
1813+
" register_audio_decode_module(registry);\n")
1814+
list(APPEND AUDIOCPP_SERVER_FRONTEND_SOURCES
1815+
app/server/frontends/audio_decode.cpp)
1816+
list(APPEND AUDIOCPP_SERVER_FRONTEND_INCLUDE_DIRS
1817+
${CMAKE_CURRENT_SOURCE_DIR}/external/miniaudio)
1818+
elseif (AUDIOCPP_SERVER_FRONTEND_MODULE STREQUAL "mp3_encode")
1819+
find_path(AUDIOCPP_LAME_INCLUDE_DIR
1820+
NAMES lame/lame.h
1821+
HINTS "$ENV{CONDA_PREFIX}"
1822+
PATH_SUFFIXES include)
1823+
find_library(AUDIOCPP_LAME_LIBRARY
1824+
NAMES mp3lame lame
1825+
HINTS "$ENV{CONDA_PREFIX}"
1826+
PATH_SUFFIXES lib)
1827+
if (NOT AUDIOCPP_LAME_INCLUDE_DIR OR NOT AUDIOCPP_LAME_LIBRARY)
1828+
message(FATAL_ERROR
1829+
"AUDIOCPP_SERVER_FRONTEND_MODULES=mp3_encode requires libmp3lame headers and library "
1830+
"(install libmp3lame-dev or build from an environment that provides lame/lame.h)")
1831+
endif()
1832+
string(APPEND AUDIOCPP_SERVER_FRONTEND_DECLARATIONS
1833+
"void register_mp3_encode_module(ServerFrontendRegistry & registry);\n")
1834+
string(APPEND AUDIOCPP_SERVER_FRONTEND_REGISTRATIONS
1835+
" register_mp3_encode_module(registry);\n")
1836+
list(APPEND AUDIOCPP_SERVER_FRONTEND_SOURCES
1837+
app/server/frontends/mp3_encode.cpp)
1838+
list(APPEND AUDIOCPP_SERVER_FRONTEND_INCLUDE_DIRS
1839+
${AUDIOCPP_LAME_INCLUDE_DIR})
1840+
list(APPEND AUDIOCPP_SERVER_FRONTEND_LIBRARIES
1841+
${AUDIOCPP_LAME_LIBRARY})
1842+
else()
1843+
message(FATAL_ERROR
1844+
"Unknown AUDIOCPP_SERVER_FRONTEND_MODULES entry: ${AUDIOCPP_SERVER_FRONTEND_MODULE}")
1845+
endif()
1846+
endforeach()
1847+
1848+
if (AUDIOCPP_SERVER_FRONTEND_SOURCES)
1849+
list(REMOVE_DUPLICATES AUDIOCPP_SERVER_FRONTEND_INCLUDE_DIRS)
1850+
list(REMOVE_DUPLICATES AUDIOCPP_SERVER_FRONTEND_LIBRARIES)
1851+
add_library(audiocpp_server_frontends OBJECT
1852+
${AUDIOCPP_SERVER_FRONTEND_SOURCES}
1853+
)
1854+
# Frontend-only dependencies belong here so default server builds do not include or link them.
1855+
target_include_directories(audiocpp_server_frontends PRIVATE
1856+
${AUDIOCPP_SERVER_FRONTEND_INCLUDE_DIRS}
1857+
)
1858+
target_link_libraries(audiocpp_server_frontends PUBLIC engine_runtime)
1859+
target_sources(audiocpp_server PRIVATE $<TARGET_OBJECTS:audiocpp_server_frontends>)
1860+
target_link_libraries(audiocpp_server PRIVATE
1861+
audiocpp_server_frontends
1862+
${AUDIOCPP_SERVER_FRONTEND_LIBRARIES})
1863+
endif()
1864+
endif()
1865+
file(WRITE
1866+
"${CMAKE_CURRENT_BINARY_DIR}/generated/server_frontend_module_declarations.inc"
1867+
"${AUDIOCPP_SERVER_FRONTEND_DECLARATIONS}")
1868+
file(WRITE
1869+
"${CMAKE_CURRENT_BINARY_DIR}/generated/server_frontend_module_registrations.inc"
1870+
"${AUDIOCPP_SERVER_FRONTEND_REGISTRATIONS}")
17921871
if (AUDIOCPP_BUILD_NATIVE_MODEL_MANAGER)
17931872
target_sources(audiocpp_server PRIVATE app/server/model_installer.cpp)
17941873
target_link_libraries(audiocpp_server PRIVATE audiocpp_package_manager)

app/server/README.md

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,23 @@ Pick the mode that matches the behavior you want:
2323
| Standalone deployed binary without local `model_specs/` | `-DAUDIOCPP_DEPLOYMENT_BUILD=ON` | `audiocpp_server --config server.json` | Binary carries compiled package specs for fallback model-spec lookup. |
2424
| Offline/reproducible native-manager build | `-DAUDIOCPP_BUILD_NATIVE_MODEL_MANAGER=ON -DAUDIOCPP_BORINGSSL_ARCHIVE=/path/to/boringssl.tar.gz` | `audiocpp_server --ui --ui-management --backend <backend>` | Configure does not fetch BoringSSL from the network. |
2525
| Distro-packaged TLS instead of bundled BoringSSL | `-DAUDIOCPP_BUILD_NATIVE_MODEL_MANAGER=ON -DAUDIOCPP_USE_SYSTEM_OPENSSL=ON` | `audiocpp_server --ui --ui-management --backend <backend>` | Uses system OpenSSL; useful for packagers. |
26+
| Optional in-process frontend pipeline | `-DAUDIOCPP_BUILD_SERVER_FRONTENDS=ON -DAUDIOCPP_SERVER_FRONTEND_MODULES="audio_decode;mp3_encode"` | `audiocpp_server --config server.json` | Adds compiled-in pre/post processing modules around the stable core API. `audio_decode` accepts MP3/FLAC transcription input through miniaudio; `mp3_encode` returns `response_format=mp3` speech output through libmp3lame. The default server build includes none of these modules or dependencies. |
2627

2728
Native model management uses bundled BoringSSL by default. Normal server builds
2829
do not build or link that HTTP/TLS dependency.
2930

31+
Optional frontend modules are selected at configure time with the semicolon-separated
32+
`AUDIOCPP_SERVER_FRONTEND_MODULES` list. The server runs selected modules as an
33+
ordered pipeline: every module gets a pre-processing pass before the core handler,
34+
then every module gets a post-processing pass after the core handler. A module that
35+
does not need one side leaves that method empty. Each active side declares a simple
36+
contract over the HTTP envelope state (`method`, `path`, `request_in/request_out`
37+
for pre-processing, or `response_in/response_out` for post-processing), and module
38+
registration rejects incompatible adjacent transforms on the same route. Add a new
39+
module by implementing `ServerFrontendModule`, declaring its contract, registering it
40+
from the generated CMake registration list, and appending its source and private
41+
dependencies in the `AUDIOCPP_SERVER_FRONTEND_MODULES` branch for that module.
42+
3043
## Config
3144

3245
```bash
@@ -319,7 +332,7 @@ curl http://127.0.0.1:8080/v1/audio/speech \
319332
}'
320333
```
321334

322-
Set `"response_format": "json"` to receive base64 WAV in a JSON response.
335+
Set `"response_format": "json"` to receive base64 WAV in a JSON response. In builds configured with `-DAUDIOCPP_BUILD_SERVER_FRONTENDS=ON -DAUDIOCPP_SERVER_FRONTEND_MODULES=mp3_encode`, `"response_format": "mp3"` returns `audio/mpeg` MP3 bytes for non-streaming speech requests.
323336

324337
For streaming-capable TTS models configured with `mode: "streaming"`, `stream_format` follows the OpenAI speech streaming shape:
325338

@@ -344,7 +357,7 @@ The SSE stream emits `speech.audio.delta` events with base64 PCM chunks, then `s
344357

345358
### `POST /v1/audio/transcriptions`
346359

347-
JSON transcription request using a server-local audio path.
360+
JSON transcription request using a server-local WAV audio path.
348361

349362
```bash
350363
curl http://127.0.0.1:8080/v1/audio/transcriptions \
@@ -364,7 +377,7 @@ curl http://127.0.0.1:8080/v1/audio/transcriptions \
364377
-F file=@/path/to/input.wav
365378
```
366379

367-
`file` and `model` are required; `language` is optional. Uploaded WAV bytes are decoded in memory and are not written to a temporary file.
380+
`file` and `model` are required; `language` is optional. Uploaded WAV bytes are decoded in memory and are not written to a temporary file. In builds configured with `-DAUDIOCPP_BUILD_SERVER_FRONTENDS=ON -DAUDIOCPP_SERVER_FRONTEND_MODULES=audio_decode`, the frontend also accepts MP3 and FLAC input for this route, decodes it to a temporary WAV, and forwards that normalized request to the same core transcription handler.
368381

369382
For streaming-capable ASR models configured with `mode: "streaming"`, pass `stream=true` to receive OpenAI-style transcription SSE:
370383

app/server/frontend.cpp

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
#include "frontend.h"
2+
3+
#include <stdexcept>
4+
#include <string>
5+
#include <utility>
6+
7+
namespace minitts::server {
8+
9+
namespace {
10+
11+
bool contract_value_matches(std::string_view lhs, std::string_view rhs) {
12+
return lhs == frontend_contracts::any || rhs == frontend_contracts::any || lhs == rhs;
13+
}
14+
15+
bool contract_route_overlaps(std::string_view lhs, std::string_view rhs) {
16+
return contract_value_matches(lhs, rhs);
17+
}
18+
19+
bool validate_pre_contracts(
20+
std::string_view previous_module,
21+
const FrontendPreContract & previous,
22+
std::string_view next_module,
23+
const FrontendPreContract & next) {
24+
if (!contract_route_overlaps(previous.method, next.method) ||
25+
!contract_route_overlaps(previous.path, next.path)) {
26+
return false;
27+
}
28+
if (!contract_value_matches(previous.request_out, next.request_in)) {
29+
throw std::runtime_error(
30+
"incompatible frontend pre-processing order: " + std::string(previous_module) +
31+
" outputs request state '" + std::string(previous.request_out) +
32+
"', but " + std::string(next_module) +
33+
" expects '" + std::string(next.request_in) + "'");
34+
}
35+
return true;
36+
}
37+
38+
bool validate_post_contracts(
39+
std::string_view previous_module,
40+
const FrontendPostContract & previous,
41+
std::string_view next_module,
42+
const FrontendPostContract & next) {
43+
if (!contract_route_overlaps(previous.method, next.method) ||
44+
!contract_route_overlaps(previous.path, next.path)) {
45+
return false;
46+
}
47+
if (!contract_value_matches(previous.response_out, next.response_in)) {
48+
throw std::runtime_error(
49+
"incompatible frontend post-processing order: " + std::string(previous_module) +
50+
" outputs response state '" + std::string(previous.response_out) +
51+
"', but " + std::string(next_module) +
52+
" expects '" + std::string(next.response_in) + "'");
53+
}
54+
return true;
55+
}
56+
57+
} // namespace
58+
59+
#include "server_frontend_module_declarations.inc"
60+
61+
std::optional<FrontendPreContract> ServerFrontendModule::pre_contract() const {
62+
return std::nullopt;
63+
}
64+
65+
std::optional<FrontendPostContract> ServerFrontendModule::post_contract() const {
66+
return std::nullopt;
67+
}
68+
69+
void ServerFrontendModule::pre_process(ServerFrontendContext & context, ServerFrontendRequest & request) {
70+
(void) context;
71+
(void) request;
72+
}
73+
74+
void ServerFrontendModule::post_process(ServerFrontendContext & context, ServerFrontendResponse & response) {
75+
(void) context;
76+
(void) response;
77+
}
78+
79+
void ServerFrontendRegistry::add(ServerFrontendModuleFactory factory) {
80+
if (factory == nullptr) {
81+
throw std::runtime_error("server frontend registration requires a module factory");
82+
}
83+
auto module = factory();
84+
if (!module) {
85+
throw std::runtime_error("server frontend module factory returned null");
86+
}
87+
88+
const auto pre = module->pre_contract();
89+
const auto post = module->post_contract();
90+
for (auto it = modules_.rbegin(); it != modules_.rend(); ++it) {
91+
if (pre.has_value()) {
92+
if (const auto previous = (*it)->pre_contract()) {
93+
if (validate_pre_contracts((*it)->name(), *previous, module->name(), *pre)) {
94+
break;
95+
}
96+
}
97+
}
98+
}
99+
for (auto it = modules_.rbegin(); it != modules_.rend(); ++it) {
100+
if (post.has_value()) {
101+
if (const auto previous = (*it)->post_contract()) {
102+
if (validate_post_contracts((*it)->name(), *previous, module->name(), *post)) {
103+
break;
104+
}
105+
}
106+
}
107+
}
108+
109+
modules_.push_back(std::move(module));
110+
}
111+
112+
bool ServerFrontendRegistry::empty() const { return modules_.empty(); }
113+
114+
HttpResponse ServerFrontendRegistry::handle(ServerFrontendContext & context, const HttpRequest & request) const {
115+
ServerFrontendRequest frontend_request{request, std::nullopt};
116+
for (const auto & module : modules_) {
117+
module->pre_process(context, frontend_request);
118+
if (frontend_request.response.has_value()) {
119+
return std::move(*frontend_request.response);
120+
}
121+
}
122+
123+
auto core_response = context.forward_to_core(frontend_request.request);
124+
ServerFrontendResponse frontend_response{request, frontend_request.request, std::move(core_response)};
125+
for (const auto & module : modules_) {
126+
module->post_process(context, frontend_response);
127+
}
128+
return std::move(frontend_response.response);
129+
}
130+
131+
void register_static_server_frontends(ServerFrontendRegistry & registry) {
132+
#include "server_frontend_module_registrations.inc"
133+
}
134+
135+
} // namespace minitts::server

app/server/frontend.h

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
#pragma once
2+
3+
#include "http.h"
4+
5+
#include <filesystem>
6+
#include <functional>
7+
#include <memory>
8+
#include <optional>
9+
#include <string_view>
10+
#include <vector>
11+
12+
namespace minitts::server {
13+
14+
class ServerFrontendContext {
15+
public:
16+
virtual ~ServerFrontendContext() = default;
17+
18+
virtual HttpResponse forward_to_core(const HttpRequest & request) = 0;
19+
virtual std::filesystem::path resolve_request_path(const std::filesystem::path & path) const = 0;
20+
virtual std::filesystem::path make_frontend_temp_path(std::string_view filename) = 0;
21+
};
22+
23+
struct ServerFrontendRequest {
24+
HttpRequest request;
25+
std::optional<HttpResponse> response;
26+
};
27+
28+
struct ServerFrontendResponse {
29+
const HttpRequest & original_request;
30+
const HttpRequest & core_request;
31+
HttpResponse response;
32+
};
33+
34+
struct FrontendPreContract {
35+
std::string_view method;
36+
std::string_view path;
37+
std::string_view request_in;
38+
std::string_view request_out;
39+
};
40+
41+
struct FrontendPostContract {
42+
std::string_view method;
43+
std::string_view path;
44+
std::string_view response_in;
45+
std::string_view response_out;
46+
};
47+
48+
namespace frontend_contracts {
49+
inline constexpr std::string_view any = "*";
50+
inline constexpr std::string_view client_encoded_audio_request = "client_encoded_audio_request";
51+
inline constexpr std::string_view core_wav_audio_request = "core_wav_audio_request";
52+
inline constexpr std::string_view client_mp3_speech_request = "client_mp3_speech_request";
53+
inline constexpr std::string_view core_wav_speech_request = "core_wav_speech_request";
54+
inline constexpr std::string_view core_wav_speech_response = "core_wav_speech_response";
55+
inline constexpr std::string_view client_mp3_speech_response = "client_mp3_speech_response";
56+
} // namespace frontend_contracts
57+
58+
class ServerFrontendModule {
59+
public:
60+
virtual ~ServerFrontendModule() = default;
61+
62+
virtual std::string_view name() const = 0;
63+
virtual std::optional<FrontendPreContract> pre_contract() const;
64+
virtual std::optional<FrontendPostContract> post_contract() const;
65+
virtual void pre_process(ServerFrontendContext & context, ServerFrontendRequest & request);
66+
virtual void post_process(ServerFrontendContext & context, ServerFrontendResponse & response);
67+
};
68+
69+
using ServerFrontendModuleFactory = std::unique_ptr<ServerFrontendModule> (*)();
70+
71+
class ServerFrontendRegistry {
72+
public:
73+
void add(ServerFrontendModuleFactory factory);
74+
bool empty() const;
75+
HttpResponse handle(ServerFrontendContext & context, const HttpRequest & request) const;
76+
77+
private:
78+
std::vector<std::unique_ptr<ServerFrontendModule>> modules_;
79+
};
80+
81+
void register_static_server_frontends(ServerFrontendRegistry & registry);
82+
83+
} // namespace minitts::server

0 commit comments

Comments
 (0)