Skip to content

feat: add RMS-based loudness normalization #262

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Philosyang wants to merge 1 commit into
base: dev
Choose a base branch
from
Draft

feat: add RMS-based loudness normalization #262

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions Assets/Scripts/IO/AudioManager.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -442,7 +442,7 @@ public void ReadVolumeFromSettings()
}
}

public AudioSampleWrap LoadMusic(string path, bool normalize = true, bool speedChange = false)
public AudioSampleWrap LoadMusic(string path, bool normalize = true, bool speedChange = false, float? cachedRmsDb = null)
{
MajDebug.LogInfo($"Try creating channel from file: {path}");
var backend = MajInstances.Settings.Audio.Backend;
Expand All @@ -456,10 +456,10 @@ public AudioSampleWrap LoadMusic(string path, bool normalize = true, bool speedC
break;
case SoundBackendOption.Asio:
case SoundBackendOption.Wasapi:
sample = BassAudioSample.Create(path, BassGlobalMixer, normalize, speedChange);
sample = BassAudioSample.Create(path, BassGlobalMixer, normalize, speedChange, cachedRmsDb);
break;
case SoundBackendOption.BassSimple:
sample = BassSimpleAudioSample.Create(path, normalize, speedChange);
sample = BassSimpleAudioSample.Create(path, normalize, speedChange, cachedRmsDb);
break;
default:
MajDebug.LogError("Backend not supported");
Expand Down Expand Up @@ -498,7 +498,7 @@ public AudioSampleWrap LoadMusicFromUri(Uri uri)
MajDebug.LogInfo("Channel created");
return sample;
}
public async UniTask<AudioSampleWrap> LoadMusicAsync(string path, bool normalize = true, bool speedChange = false)
public async UniTask<AudioSampleWrap> LoadMusicAsync(string path, bool normalize = true, bool speedChange = false, float? cachedRmsDb = null)
{
MajDebug.LogInfo($"Try creating channel from file: {path}");
await UniTask.SwitchToThreadPool();
Expand All @@ -514,10 +514,10 @@ public async UniTask<AudioSampleWrap> LoadMusicAsync(string path, bool normalize
break;
case SoundBackendOption.Asio:
case SoundBackendOption.Wasapi:
sample = await BassAudioSample.CreateAsync(path, BassGlobalMixer, normalize, speedChange);
sample = await BassAudioSample.CreateAsync(path, BassGlobalMixer, normalize, speedChange, cachedRmsDb);
break;
case SoundBackendOption.BassSimple:
sample = await BassSimpleAudioSample.CreateAsync(path, normalize, speedChange);
sample = await BassSimpleAudioSample.CreateAsync(path, normalize, speedChange, cachedRmsDb);
break;
default:
MajDebug.LogError("Backend not supported");
Expand Down
3 changes: 2 additions & 1 deletion Assets/Scripts/IO/Base/Audio/AudioSampleWrap.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
using System;
using System;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
Expand All @@ -22,6 +22,7 @@ public abstract class AudioSampleWrap : IDisposable, IPausableSoundProvider
public abstract TimeSpan Length { get; }
public abstract bool IsLoop { get; set; }
public bool CanSeek { get; protected init; }
public float? MeasuredRmsDb { get; set; } = null;

protected bool _isDisposed = false;

Expand Down
53 changes: 46 additions & 7 deletions Assets/Scripts/IO/Base/Audio/BassAudioSample.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,11 @@ public override ValueTask DisposeAsync()

return new ValueTask(Task.CompletedTask);
}
static BassAudioSample Create(byte[] data, int globalMixer, bool normalize, bool speedChange)
const double TARGET_RMS_DB = -14.0;
const double GAIN_MIN = 0.1;
const double GAIN_MAX = 4.0;

static BassAudioSample Create(byte[] data, int globalMixer, bool normalize, bool speedChange, float? cachedRmsDb = null)
{
var handle = GCHandle.Alloc(data, GCHandleType.Pinned);
var addr = handle.AddrOfPinnedObject();
Expand All @@ -244,10 +248,44 @@ static BassAudioSample Create(byte[] data, int globalMixer, bool normalize, bool
}
Bass.ChannelSetAttribute(decode, ChannelAttribute.Buffer, 0);

//scan the peak here
var bytelength = Bass.ChannelGetLength(decode);
var gain = 1d;
if (normalize)
float? measuredRmsDb = null;
var useLoudnessNorm = MajInstances.Settings.Audio.LoudnessNormalization;

if (normalize && useLoudnessNorm && cachedRmsDb.HasValue)
{
var rmsDb = (double)cachedRmsDb.Value;
gain = Math.Pow(10, (TARGET_RMS_DB - rmsDb) / 20.0);
gain = Math.Clamp(gain, GAIN_MIN, GAIN_MAX);
measuredRmsDb = cachedRmsDb.Value;
MajDebug.LogInfo($"[LoudnessNorm] Using cached RMS: {rmsDb:F2} dB, gain: {gain:F4}");
}
else if (normalize && useLoudnessNorm)
{
double sumSquares = 0;
int windowCount = 0;
while (Bass.ChannelGetPosition(decode, PositionFlags.Decode | PositionFlags.Bytes) < bytelength)
{
var levels = Bass.ChannelGetLevel(decode, 1.0f, LevelRetrievalFlags.RMS | LevelRetrievalFlags.Mono);
if (levels is not null && levels.Length > 0)
{
double rms = levels[0];
sumSquares += rms * rms;
windowCount++;
}
}
if (windowCount > 0)
{
var avgRms = Math.Sqrt(sumSquares / windowCount);
var rmsDb = 20.0 * Math.Log10(Math.Max(avgRms, 1e-10));
gain = Math.Pow(10, (TARGET_RMS_DB - rmsDb) / 20.0);
gain = Math.Clamp(gain, GAIN_MIN, GAIN_MAX);
measuredRmsDb = (float)rmsDb;
MajDebug.LogInfo($"[LoudnessNorm] Measured RMS: {rmsDb:F2} dB, target: {TARGET_RMS_DB:F2} dB, gain: {gain:F4}");
}
}
else if (normalize)
{
double channelmax = 0;
while (Bass.ChannelGetPosition(decode, PositionFlags.Decode | PositionFlags.Bytes) < bytelength)
Expand All @@ -264,6 +302,7 @@ static BassAudioSample Create(byte[] data, int globalMixer, bool normalize, bool
var sample = new BassAudioSample(decode, globalMixer, gain, speedChange)
{
CanSeek = true,
MeasuredRmsDb = measuredRmsDb,
};
sample.Volume = 1;

Expand All @@ -279,17 +318,17 @@ static BassAudioSample Create(byte[] data, int globalMixer, bool normalize, bool
throw;
}
}
public static BassAudioSample Create(string path, int globalMixer, bool normalize = true, bool speedChange = false)
public static BassAudioSample Create(string path, int globalMixer, bool normalize = true, bool speedChange = false, float? cachedRmsDb = null)
{
var buf = File.ReadAllBytes(path);

return Create(buf, globalMixer, normalize, speedChange);
return Create(buf, globalMixer, normalize, speedChange, cachedRmsDb);
}
public static async ValueTask<BassAudioSample> CreateAsync(string path, int globalMixer, bool normalize = true, bool speedChange = false)
public static async ValueTask<BassAudioSample> CreateAsync(string path, int globalMixer, bool normalize = true, bool speedChange = false, float? cachedRmsDb = null)
{
var buf = await File.ReadAllBytesAsync(path);

return Create(buf, globalMixer, normalize, speedChange);
return Create(buf, globalMixer, normalize, speedChange, cachedRmsDb);
}
public static BassAudioSample CreateFromUri(Uri uri, int globalMixer)
{
Expand Down
54 changes: 47 additions & 7 deletions Assets/Scripts/IO/Base/Audio/BassSimpleAudioSample.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,11 @@ void OnAppFocus(object? sender, bool isFocus)
}
#endif
}
static BassSimpleAudioSample Create(byte[] data, bool normalize, bool speedChange)
const double TARGET_RMS_DB = -14.0;
const double GAIN_MIN = 0.1;
const double GAIN_MAX = 4.0;

static BassSimpleAudioSample Create(byte[] data, bool normalize, bool speedChange, float? cachedRmsDb = null)
{
var handle = GCHandle.Alloc(data, GCHandleType.Pinned);
var addr = handle.AddrOfPinnedObject();
Expand All @@ -227,10 +231,45 @@ static BassSimpleAudioSample Create(byte[] data, bool normalize, bool speedChang
stream = BassFx.TempoCreate(decode, BassFlags.Default);
Bass.LastError.EnsureSuccessStatusCode();
Bass.ChannelSetAttribute(stream, ChannelAttribute.Buffer, 0);
//scan the peak here

var bytelength = Bass.ChannelGetLength(decode);
var gain = 1d;
if (normalize)
float? measuredRmsDb = null;
var useLoudnessNorm = MajInstances.Settings.Audio.LoudnessNormalization;

if (normalize && useLoudnessNorm && cachedRmsDb.HasValue)
{
var rmsDb = (double)cachedRmsDb.Value;
gain = Math.Pow(10, (TARGET_RMS_DB - rmsDb) / 20.0);
gain = Math.Clamp(gain, GAIN_MIN, GAIN_MAX);
measuredRmsDb = cachedRmsDb.Value;
MajDebug.LogInfo($"[LoudnessNorm] Using cached RMS: {rmsDb:F2} dB, gain: {gain:F4}");
}
else if (normalize && useLoudnessNorm)
{
double sumSquares = 0;
int windowCount = 0;
while (Bass.ChannelGetPosition(decode, PositionFlags.Decode | PositionFlags.Bytes) < bytelength)
{
var levels = Bass.ChannelGetLevel(decode, 1.0f, LevelRetrievalFlags.RMS | LevelRetrievalFlags.Mono);
if (levels is not null && levels.Length > 0)
{
double rms = levels[0];
sumSquares += rms * rms;
windowCount++;
}
}
if (windowCount > 0)
{
var avgRms = Math.Sqrt(sumSquares / windowCount);
var rmsDb = 20.0 * Math.Log10(Math.Max(avgRms, 1e-10));
gain = Math.Pow(10, (TARGET_RMS_DB - rmsDb) / 20.0);
gain = Math.Clamp(gain, GAIN_MIN, GAIN_MAX);
measuredRmsDb = (float)rmsDb;
MajDebug.LogInfo($"[LoudnessNorm] Measured RMS: {rmsDb:F2} dB, target: {TARGET_RMS_DB:F2} dB, gain: {gain:F4}");
}
}
else if (normalize)
{
double channelmax = 0;
while (Bass.ChannelGetPosition(decode, PositionFlags.Decode | PositionFlags.Bytes) < bytelength)
Expand All @@ -247,6 +286,7 @@ static BassSimpleAudioSample Create(byte[] data, bool normalize, bool speedChang
var sample = new BassSimpleAudioSample(stream, gain, handle, speedChange)
{
CanSeek = true,
MeasuredRmsDb = measuredRmsDb,
};
sample.Volume = 1;
sample._decode = decode;
Expand All @@ -262,17 +302,17 @@ static BassSimpleAudioSample Create(byte[] data, bool normalize, bool speedChang
throw;
}
}
public static BassSimpleAudioSample Create(string path, bool normalize = true, bool speedChange = false)
public static BassSimpleAudioSample Create(string path, bool normalize = true, bool speedChange = false, float? cachedRmsDb = null)
{
var buf = File.ReadAllBytes(path);

return Create(buf, normalize, speedChange);
return Create(buf, normalize, speedChange, cachedRmsDb);
}
public static async ValueTask<BassSimpleAudioSample> CreateAsync(string path, bool normalize = true, bool speedChange = false)
public static async ValueTask<BassSimpleAudioSample> CreateAsync(string path, bool normalize = true, bool speedChange = false, float? cachedRmsDb = null)
{
var buf = await File.ReadAllBytesAsync(path);

return Create(buf, normalize, speedChange);
return Create(buf, normalize, speedChange, cachedRmsDb);
}
public static BassSimpleAudioSample CreateFromUri(Uri uri)
{
Expand Down
7 changes: 6 additions & 1 deletion Assets/Scripts/Misc/Base/DataFormats/OnlineSongDetail.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -534,7 +534,8 @@ async ValueTask<AudioSampleWrap> GetAudioTrackAsync(bool loadIntoMemory, INetPro
{
return AudioSampleWrap.Empty;
}
var sampleWarp = await MajInstances.AudioManager.LoadMusicAsync(savePath, true, true);
var cachedRmsDb = _chartSettings.MeasuredRmsDb;
var sampleWarp = await MajInstances.AudioManager.LoadMusicAsync(savePath, true, true, cachedRmsDb);
if (sampleWarp.IsEmpty)
{
if (File.Exists(cacheFlagPath))
Expand All @@ -543,6 +544,10 @@ async ValueTask<AudioSampleWrap> GetAudioTrackAsync(bool loadIntoMemory, INetPro
}
await DownloadFile(_trackUri, savePath, false, progress, token);
}
if (sampleWarp.MeasuredRmsDb.HasValue && sampleWarp.MeasuredRmsDb != cachedRmsDb)
{
_chartSettings.MeasuredRmsDb = sampleWarp.MeasuredRmsDb;
}
_audioTrackRef.SetTarget(sampleWarp);

return sampleWarp;
Expand Down
7 changes: 6 additions & 1 deletion Assets/Scripts/Misc/Base/DataFormats/SongDetail.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,12 @@ public async ValueTask<AudioSampleWrap> GetAudioTrackAsync(INetProgress? progres
return audioTrack;
}
progress?.Report(1);
audioTrack = await MajInstances.AudioManager.LoadMusicAsync(_trackPath, true, true);
var cachedRmsDb = _chartSettings.MeasuredRmsDb;
audioTrack = await MajInstances.AudioManager.LoadMusicAsync(_trackPath, true, true, cachedRmsDb);
if (audioTrack.MeasuredRmsDb.HasValue && audioTrack.MeasuredRmsDb != cachedRmsDb)
{
_chartSettings.MeasuredRmsDb = audioTrack.MeasuredRmsDb;
}
_audioTrackRef.SetTarget(audioTrack);
return audioTrack;
}
Expand Down
4 changes: 3 additions & 1 deletion Assets/Scripts/Misc/Settings/ChartSetting.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
Expand All @@ -21,5 +21,7 @@ public class ChartSetting
[Range("-2", "2", HasMax = true, HasMin = true)]
public float TrackVolumeOffset { get; set; } = 0f;
public bool DisableVideoBG { get; set; } = false;
[HideInSettingUI, Preserve]
public float? MeasuredRmsDb { get; set; } = null;
}
}
2 changes: 2 additions & 0 deletions Assets/Scripts/Misc/Settings/GameSetting.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,8 @@ public class SoundOptions
#endif

public SoundBackendOption Backend { get; set; } = DEFAULT_SOUND_BACKEND;

public bool LoudnessNormalization { get; set; } = false;
}

public class SFXVolume
Expand Down