-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathPodcastMixer.cs
More file actions
474 lines (435 loc) · 20.6 KB
/
Copy pathPodcastMixer.cs
File metadata and controls
474 lines (435 loc) · 20.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
using System.Runtime.InteropServices;
using System.Text.RegularExpressions;
using GroovyCodecs.Mp3;
using GroovyCodecs.Types;
namespace AIOrchestrator.API;
/// <summary>Internal: renders the podcast audio. The narration (welcome intro first, then the
/// body with its [markers] as silences/pauses) is synthesized sentence by sentence with the
/// plugin-local <see cref="PodcastTts"/> engine; the CC0 jingle and background loop are
/// decoded and resampled to 48 kHz stereo with OwnAudioSharp (FileSource); the streams are
/// mixed sample by sample (deterministic, fast, cross-platform) with a DUCKED background bed
/// (barely perceptible under the voice, rising in the pauses), then encoded to MP3 with the
/// pure-managed GroovyMp3 (LAME) encoder.</summary>
internal static partial class PodcastMixer
{
/// <summary>Human-readable reason when <see cref="Mix"/> fails (kept for the tool result).</summary>
internal static string LastError { get; private set; } = "";
// Background levels: a soft bed that is "barely perceptible" under the voice and rises
// slightly during the silences (ducking, deterministic at sample level).
private const float BackgroundWhileSpeaking = 0.05f;
private const float BackgroundInPause = 0.12f;
/// <summary>Mixes the episode (intro → jingle → body) and encodes the MP3. Returns false
/// when the TTS engine or the music assets are unavailable (LastError explains why);
/// <paramref name="durationSeconds"/> receives the actual audio length (mixing + silences,
/// from the rendered PCM) for the RSS feed.</summary>
internal static bool Mix(string intro, string body, string lang, string wavPath, string mp3Path, out double durationSeconds)
{
durationSeconds = 0;
LastError = "";
if (!PodcastTts.IsAvailable)
{
LastError = PodcastTts.UnavailableReason;
return false;
}
EnsureNativeAvailable();
if (!TryLoadMusic(out var jingle, out var background))
return false;
var sampleRate = PodcastTts.SampleRate; // 24 kHz mono narration
const int outRate = 48000; // output: 48 kHz stereo
// jingle is short[] stereo: frames = Length/2, seconds = frames/outRate. (A spurious
// extra /2 here halved the slot and let the first act overlap the jingle — fixed.)
var jingleDuration = (double)jingle.Length / 2 / outRate; // stereo int16 frames → seconds
// 1) Narration = welcome intro + 0.6 s breath + the body (markers become silences).
// The jingle slot is the gap right after the intro.
var narration = new List<byte>(sampleRate * 2 * 60); // int16 mono
var currentSec = 0.0;
var jingleStartSec = -1.0;
var stingerStartSec = -1.0;
void AppendPcm(byte[] pcm)
{
narration.AddRange(pcm);
currentSec += (double)pcm.Length / 2 / sampleRate;
}
void AppendSilence(double seconds)
{
var n = (int)(seconds * sampleRate);
for (int i = 0; i < n; i++)
{
narration.Add(0);
narration.Add(0);
}
currentSec += seconds;
}
AppendPcm(Speak(intro, lang));
AppendSilence(0.6);
jingleStartSec = currentSec;
AppendSilence(jingleDuration);
foreach (var token in Regex.Split(body, @"(\[[^\]]+\])"))
{
if (string.IsNullOrWhiteSpace(token)) continue;
var marker = Regex.Match(token, @"^\[([^\]]+)\]$");
if (marker.Success)
{
var name = marker.Groups[1].Value.Trim().ToLowerInvariant();
switch (name)
{
case "sigla":
AppendSilence(0.8); // the jingle is placed by the mixer, not the marker
break;
case "pausa musicale":
AppendSilence(0.8);
break;
case "breve silenzio":
AppendSilence(0.4);
break;
case "primo atto":
case "secondo atto":
case "terzo atto":
case "outro":
AppendSilence(1.0);
break;
default:
AppendSilence(0.3);
break;
}
continue;
}
AppendPcm(Speak(token, lang));
}
// 1b) The closing stinger: a short final musical hit right after the outro speech
// (0.6 s breath, then the last ~7 s of the sigla with a fade-out).
var stingerDuration = Math.Min(7.0, jingleDuration);
AppendSilence(0.6);
stingerStartSec = currentSec;
AppendSilence(stingerDuration);
if (narration.Count == 0)
{
LastError = "the script contains no speakable text.";
return false;
}
Log.LogStep($"PodcastMixer: narration {(double)narration.Count / 2 / sampleRate:F0}s, jingle at {jingleStartSec:F0}s (jingle {jingleDuration:F0}s), stinger at {stingerStartSec:F0}s");
// 2) Mix: narration (int16 mono 24 kHz → 48 kHz stereo) + jingle at its slot (fade
// out) + ducked background loop.
var narration16 = MemoryMarshal.Cast<byte, short>(CollectionsMarshal.AsSpan(narration));
int narrationFrames24 = narration16.Length;
long totalFrames48 = (long)narrationFrames24 * 2;
var jingleStartFrame = jingleStartSec >= 0 ? (long)(jingleStartSec * outRate) : -1L;
var bodyStartFrame = jingleStartFrame >= 0 ? jingleStartFrame + jingle.Length / 2 : 0L; // the bed starts with the body
// The closing stinger: the LAST ~3 s of the sigla, played after the outro with a fade-out.
var stingerLen = (int)(stingerDuration * outRate);
var stingerOffset = jingle.Length / 2 - stingerLen; // slice index into the jingle
var stingerStartFrame = stingerStartSec >= 0 ? (long)(stingerStartSec * outRate) : -1L;
var fadeOutSamples = (int)(outRate * 0.6);
var bgFadeIn = (int)(outRate * 0.5);
var bgFadeOut = (int)(outRate * 1.0);
const float voiceThreshold = 0.008f; // what counts as "speaking"
using (var fs = File.Create(wavPath))
using (var bw = new BinaryWriter(fs))
{
WriteWavHeader(bw, totalFrames48, outRate);
var block = new short[outRate * 2];
var bgGain = BackgroundWhileSpeaking; // smoothed duck level
for (long f = 0; f < totalFrames48;)
{
var n = (int)Math.Min(outRate, totalFrames48 - f);
for (int k = 0; k < n; k++)
{
var idx = f + k;
var nIdx = (int)(idx >> 1);
if (nIdx >= narration16.Length) break;
var voice = narration16[nIdx] / 32768f;
// Ducking: the bed exists only under the body — the intro and the jingle
// are clean. Under the voice it drops to "barely perceptible", rising in
// the pauses. The closing stinger replaces the bed entirely.
var g = 0f;
if (idx >= bodyStartFrame)
{
if (stingerStartFrame >= 0 && idx >= stingerStartFrame)
g = 0; // the final stinger is clean
else
{
var target = Math.Abs(voice) > voiceThreshold ? BackgroundWhileSpeaking : BackgroundInPause;
bgGain += (target - bgGain) * 0.01f; // smooth attack/release
g = bgGain;
var t = idx - bodyStartFrame;
if (t < bgFadeIn) g *= t / (float)bgFadeIn;
if (idx > totalFrames48 - bgFadeOut) g *= (totalFrames48 - idx) / (float)bgFadeOut;
}
}
var bf = (int)(idx % (background.Length / 2));
float bL = background[bf * 2] / 32768f * g;
float bR = background[bf * 2 + 1] / 32768f * g;
float jL = 0f, jR = 0f;
if (jingleStartFrame >= 0)
{
var jf = idx - jingleStartFrame;
if (jf >= 0 && jf < jingle.Length / 2)
{
var jg = 1f;
if (jf > jingle.Length / 2 - fadeOutSamples)
jg = (jingle.Length / 2 - jf) / (float)fadeOutSamples;
jL = jingle[(int)jf * 2] / 32768f * jg;
jR = jingle[(int)jf * 2 + 1] / 32768f * jg;
}
}
// The closing stinger: the last ~3 s of the sigla, fading out.
float sL = 0f, sR = 0f;
if (stingerStartFrame >= 0)
{
var sfs = idx - stingerStartFrame;
if (sfs >= 0 && sfs < stingerLen)
{
var sg = 1f;
if (sfs > stingerLen - fadeOutSamples)
sg = (stingerLen - sfs) / (float)fadeOutSamples;
sL = jingle[(stingerOffset + sfs) * 2] / 32768f * sg;
sR = jingle[(stingerOffset + sfs) * 2 + 1] / 32768f * sg;
}
}
var l = voice + bL + jL + sL;
var r = voice + bR + jR + sR;
block[k * 2] = (short)Math.Clamp((int)(l * 32768f), short.MinValue, short.MaxValue);
block[k * 2 + 1] = (short)Math.Clamp((int)(r * 32768f), short.MinValue, short.MaxValue);
}
for (int i = 0; i < n * 2; i++) bw.Write(block[i]);
f += n;
}
bw.Flush();
} // WAV closed here — the MP3 encoder can now read it
// 3) MP3: encode the mixed WAV with the pure-managed GroovyMp3 (LAME) encoder, then
// drop the bulky WAV intermediate (the MP3 is the deliverable).
durationSeconds = (double)totalFrames48 / outRate;
if (!EncodeMp3(wavPath, mp3Path)) return false;
try { File.Delete(wavPath); } catch { }
return true;
}
/// <summary>Synthesizes a text chunk to int16 PCM (never null — empty for unreadable text).</summary>
private static byte[] Speak(string text, string lang)
{
// The split and the speakable conversion are plugin-local (PodcastTts.SplitSpeakable):
// binding the host's VoiceConversation helpers here broke older plugin builds when
// their signatures drifted (MissingMethodException at mix time).
var pcm = new List<byte>();
foreach (var sentence in PodcastTts.SplitSpeakable(text))
{
var bytes = PodcastTts.SynthesizePcm(sentence, lang);
if (bytes != null && bytes.Length > 0) pcm.AddRange(bytes);
}
return pcm.ToArray();
}
/// <summary>Encodes a 48 kHz stereo 16-bit WAV to MP3 with GroovyMp3 (pure managed LAME
/// port — cross-platform, no natives), feeding the PCM in chunks and flushing at the end.
/// Internal: the test harness uses it to recover a mixed WAV whose MP3 encode step failed
/// (missing assembly in a stale incremental output) without re-synthesizing.</summary>
internal static bool EncodeMp3(string wavPath, string mp3Path)
{
try
{
var pcm = File.ReadAllBytes(wavPath);
if (pcm.Length <= 44) return Fail("the mixed WAV is empty.");
var encoder = new Mp3Encoder();
var format = new AudioFormat
{
SampleRate = 48000,
Channels = 2,
BitsPerSample = 16,
BigEndian = false,
IsFloatingPoint = false,
BlockAlign = 4,
AverageBytesPerSecond = 48000 * 2 * 2,
};
encoder.SetFormat(format, format);
using var outFile = File.Create(mp3Path);
var chunkSize = Math.Min(encoder.InputBufferSize > 0 ? encoder.InputBufferSize : 46080, 46080);
var inBuf = new byte[chunkSize];
var outBuf = new byte[encoder.OutputBufferSize > 0 ? encoder.OutputBufferSize : chunkSize + 4096];
int offset = 44; // skip the WAV header
while (offset < pcm.Length)
{
var take = Math.Min(chunkSize, pcm.Length - offset);
Buffer.BlockCopy(pcm, offset, inBuf, 0, take);
var encoded = encoder.EncodeBuffer(inBuf, 0, take, outBuf);
if (encoded > 0) outFile.Write(outBuf, 0, encoded);
offset += take;
}
var tail = encoder.EncodeFinish(outBuf);
if (tail > 0) outFile.Write(outBuf, 0, tail);
encoder.Close();
var size = new FileInfo(mp3Path).Length;
if (size < 1024) return Fail($"the MP3 encode produced no audio ({size} bytes).");
Log.LogStep($"PodcastMixer: MP3 encoded — {size / 1024d / 1024d:F1} MB");
return true;
}
catch (Exception ex)
{
return Fail($"the MP3 encode failed: {ex.Message}");
}
}
private static bool Fail(string reason)
{
LastError = reason;
return false;
}
/// <summary>Decodes the jingle and the background loop to 48 kHz stereo int16 with
/// OwnAudioSharp (jingle: first 12 seconds — a full CC0 track is longer than a sigla).</summary>
private static bool TryLoadMusic(out short[] jingle, out short[] background)
{
jingle = Array.Empty<short>();
background = Array.Empty<short>();
var jinglePath = FindAsset("jingle.mp3");
var bgPath = FindAsset("background.mp3");
if (jinglePath == null || bgPath == null)
{
LastError = "the CC0 music assets (jingle/background) are not deployed. Reinstall the plugin.";
return false;
}
try
{
var jingleFile = new OwnaudioNET.Sources.FileSource(jinglePath, 8192, 48000, 2);
jingle = ReadAllSamples(jingleFile, maxSeconds: 12);
jingleFile.Dispose();
var bgFile = new OwnaudioNET.Sources.FileSource(bgPath, 8192, 48000, 2);
background = ReadAllSamples(bgFile, maxSeconds: 0);
bgFile.Dispose();
if (jingle.Length < 48000 || background.Length < 48000)
{
LastError = "the music assets are empty or unreadable.";
return false;
}
return true;
}
catch (Exception ex)
{
LastError = $"the music assets failed to load: {ex.Message}";
return false;
}
}
/// <summary>Reads an entire source into a flat stereo int16 array (resampled to 48 kHz
/// stereo by the FileSource). Cap the jingle length; the background loop is bounded too
/// (a hard frame cap keeps the read terminating even if the decoder never signals the
/// end of stream).</summary>
private static short[] ReadAllSamples(OwnaudioNET.Sources.FileSource source, int maxSeconds)
{
var capSeconds = maxSeconds > 0 ? maxSeconds : 180; // 3 min hard cap for the loop
var samples = new List<short>(capSeconds * 48000 * 2);
var buffer = new float[4096 * 2];
var maxFrames = capSeconds * 48000;
var totalFrames = 0;
source.Play();
while (!source.IsEndOfStream && totalFrames < maxFrames)
{
var frames = source.ReadSamples(buffer.AsSpan(), 4096);
if (frames <= 0) break;
for (int i = 0; i < frames * 2; i++)
samples.Add((short)Math.Clamp((int)(buffer[i] * 32768f), short.MinValue, short.MaxValue));
totalFrames += frames;
}
return samples.ToArray();
}
/// <summary>Locates a music asset: the host assets/ folder (PluginUpdater merge target),
/// the plugin's own assets/audio/ folder, or — last resort — the embedded resource copy
/// (the dev ship target excludes assets/, so the embedded copy is the guarantee).</summary>
private static string? FindAsset(string fileName)
{
var candidates = new[]
{
Path.Combine(AppContext.BaseDirectory, "assets", "audio", fileName),
Path.Combine(AppContext.BaseDirectory, "assets", fileName),
Path.Combine(AppContext.BaseDirectory, fileName),
};
foreach (var c in candidates)
{
if (File.Exists(c)) return c;
}
var toolsDir = Path.Combine(AppContext.BaseDirectory, "Tools");
if (Directory.Exists(toolsDir))
{
foreach (var dir in Directory.GetDirectories(toolsDir, "*", SearchOption.AllDirectories))
{
var candidate = Path.Combine(dir, "assets", "audio", fileName);
if (File.Exists(candidate)) return candidate;
}
}
try
{
var asm = typeof(PodcastMixer).Assembly;
var logical = "PodcastTool.assets.audio." + fileName;
using var stream = asm.GetManifestResourceStream(logical);
if (stream == null) return null;
var dir = Path.Combine(Path.GetTempPath(), "podcasttool-assets");
Directory.CreateDirectory(dir);
var target = Path.Combine(dir, fileName);
if (!File.Exists(target))
{
using var dst = File.Create(target);
stream.CopyTo(dst);
}
return target;
}
catch
{
return null;
}
}
/// <summary>OwnAudioSharp's native FFI must sit in the app base directory (verified
/// empirically: the byte-loaded shim does not see the plugin folder). Copies the matching
/// per-RID native from the plugin payload on first use — idempotent, cross-platform.</summary>
private static void EnsureNativeAvailable()
{
var appBase = AppContext.BaseDirectory;
var nativeName = OperatingSystem.IsWindows() ? "ownaudio_ffi.dll"
: OperatingSystem.IsMacOS() ? "libownaudio_ffi.dylib"
: "libownaudio_ffi.so";
if (File.Exists(Path.Combine(appBase, nativeName))) return;
var rid = OperatingSystem.IsWindows()
? (RuntimeInformation.ProcessArchitecture == Architecture.Arm64 ? "win-arm64" : "win-x64")
: OperatingSystem.IsMacOS()
? (RuntimeInformation.ProcessArchitecture == Architecture.Arm64 ? "osx-arm64" : "osx-x64")
: (RuntimeInformation.ProcessArchitecture == Architecture.Arm64 ? "linux-arm64" : "linux-x64");
var dirs = new List<string>();
var toolsDir = Path.Combine(appBase, "Tools");
if (Directory.Exists(toolsDir))
{
dirs.Add(toolsDir);
dirs.AddRange(Directory.GetDirectories(toolsDir, "*", SearchOption.AllDirectories));
}
dirs.Add(appBase); // harness/dev scenario: the SDK copied runtimes/ into the bin
foreach (var dir in dirs)
{
// Probe both the runtimes/{rid}/native layout AND the flat plugin root (some
// packages drop the native flat into the output).
var candidates = new[]
{
Path.Combine(dir, "runtimes", rid, "native", nativeName),
Path.Combine(dir, nativeName),
};
foreach (var candidate in candidates)
{
if (!File.Exists(candidate)) continue;
try
{
File.Copy(candidate, Path.Combine(appBase, nativeName), overwrite: true);
return;
}
catch { }
}
}
}
private static void WriteWavHeader(BinaryWriter bw, long totalFrames48, int sampleRate)
{
var dataLen = (int)(totalFrames48 * 2 * 2); // stereo int16 — 4-byte fields in the header
bw.Write("RIFF"u8);
bw.Write(36 + dataLen);
bw.Write("WAVE"u8);
bw.Write("fmt "u8);
bw.Write(16);
bw.Write((short)1);
bw.Write((short)2);
bw.Write(sampleRate);
bw.Write(sampleRate * 2 * 2);
bw.Write((short)4);
bw.Write((short)16);
bw.Write("data"u8);
bw.Write(dataLen);
}
}