DeepTemplateMatcher.cs

using Microsoft.ML.OnnxRuntime;
using Microsoft.ML.OnnxRuntime.Tensors;
using NAudio.Dsp;

namespace VoiceCommander;

/// <summary>
/// Experimental few-shot matcher based on EfficientWord-Net's multilingual
/// ResNet-50 ArcFace embedding model. Audio is converted to 64-bin log filterbanks,
/// then compared with the user's enrollment vectors using cosine similarity.
/// </summary>
public sealed class DeepTemplateMatcher : IDisposable
{
    private const int SampleRate = 16000;
    private const int WindowSamples = 24000; // 1.5 seconds
    private const int FrameLength = 400;
    private const int FrameStep = 160;
    private const int FftSize = 512;
    private const int FilterCount = 64;
    private const int FrameCount = 149;
    private readonly SemaphoreSlim _lock = new(1, 1);
    private InferenceSession? _session;
    private string _inputName = "";
    private string _outputName = "";

    public bool IsAvailable => _session is not null;

    public async Task InitializeAsync()
    {
        var modelPath = Path.Combine(AppContext.BaseDirectory, "DeepModel", "efficientword_resnet50_qint8.onnx");
        if (!File.Exists(modelPath)) return;
        await Task.Run(() =>
        {
            var options = new SessionOptions
            {
                InterOpNumThreads = 1,
                IntraOpNumThreads = Math.Clamp(Environment.ProcessorCount / 2, 1, 4),
                GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_ALL
            };
            _session = new InferenceSession(modelPath, options);
            _inputName = _session.InputMetadata.Keys.First();
            _outputName = _session.OutputMetadata.Keys.First();
        });
    }

    public async Task<float[]> CreateEmbeddingAsync(float[] audio)
    {
        if (_session is null) throw new InvalidOperationException("深層照合モデルがありません。");
        await _lock.WaitAsync();
        try { return await Task.Run(() => CreateEmbedding(audio)); }
        finally { _lock.Release(); }
    }

    public static (VoiceCommand? Command, float Score) FindBest(float[] embedding, IEnumerable<VoiceCommand> commands)
    {
        VoiceCommand? best = null;
        var bestScore = 0f;
        foreach (var command in commands)
        {
            var scores = command.DeepTemplates
                .Where(t => t is { Length: > 0 })
                .Select(t => Similarity(embedding, t))
                .OrderByDescending(x => x)
                .ToArray();
            if (scores.Length == 0) continue;
            // One lucky enrollment should not dominate: average the best three when possible.
            var score = scores.Take(Math.Min(3, scores.Length)).Average();
            if (score > bestScore) { bestScore = score; best = command; }
        }
        return (best, bestScore);
    }

    private float[] CreateEmbedding(float[] source)
    {
        var audio = FitAndNormalize(source);
        var features = LogFilterBanks(audio);
        var tensor = new DenseTensor<float>(features, [1, 1, FrameCount, FilterCount]);
        var inputs = new List<NamedOnnxValue> { NamedOnnxValue.CreateFromTensor(_inputName, tensor) };
        using var results = _session!.Run(inputs, [_outputName]);
        var output = results.First().AsEnumerable<float>().ToArray();
        Normalize(output);
        return output;
    }

    private static float[] FitAndNormalize(float[] source)
    {
        var result = new float[WindowSamples];
        if (source.Length == 0) return result;

        // Center a 1.5 s crop on the loudest 200 ms region. Short utterances are centered.
        var block = Math.Min(3200, source.Length);
        var bestStart = 0;
        double bestEnergy = -1;
        for (var start = 0; start + block <= source.Length; start += 800)
        {
            double energy = 0;
            for (var i = start; i < start + block; i++) energy += source[i] * source[i];
            if (energy > bestEnergy) { bestEnergy = energy; bestStart = start; }
        }
        var center = bestStart + block / 2;
        var sourceStart = Math.Clamp(center - WindowSamples / 2, 0, Math.Max(0, source.Length - WindowSamples));
        var count = Math.Min(WindowSamples, source.Length - sourceStart);
        var destinationStart = (WindowSamples - count) / 2;
        Array.Copy(source, sourceStart, result, destinationStart, count);

        var peak = result.Max(x => Math.Abs(x));
        if (peak > 0.0001f)
        {
            var scale = Math.Min(12f, 0.95f / peak);
            for (var i = 0; i < result.Length; i++) result[i] *= scale;
        }
        return result;
    }

    private static float[] LogFilterBanks(float[] audio)
    {
        var output = new float[FrameCount * FilterCount];
        var bins = CreateMelBins();
        var fft = new Complex[FftSize];
        var power = new float[FftSize / 2 + 1];

        for (var frame = 0; frame < FrameCount; frame++)
        {
            Array.Clear(fft);
            var offset = frame * FrameStep;
            for (var i = 0; i < FrameLength && offset + i < audio.Length; i++) fft[i].X = audio[offset + i];
            FastFourierTransform.FFT(true, 9, fft);
            for (var i = 0; i < power.Length; i++)
                power[i] = (fft[i].X * fft[i].X + fft[i].Y * fft[i].Y) / FftSize;

            for (var filter = 0; filter < FilterCount; filter++)
            {
                double energy = 0;
                var left = bins[filter]; var middle = bins[filter + 1]; var right = bins[filter + 2];
                for (var i = left; i < middle; i++)
                    energy += power[i] * (i - left) / Math.Max(1.0, middle - left);
                for (var i = middle; i < right; i++)
                    energy += power[i] * (right - i) / Math.Max(1.0, right - middle);
                // python_speech_features uses numpy.finfo(float).eps before log().
                output[frame * FilterCount + filter] = (float)Math.Log(Math.Max(2.220446049250313e-16, energy));
            }
        }
        return output;
    }

    private static int[] CreateMelBins()
    {
        static double HzToMel(double hz) => 2595 * Math.Log10(1 + hz / 700);
        static double MelToHz(double mel) => 700 * (Math.Pow(10, mel / 2595) - 1);
        var low = HzToMel(0); var high = HzToMel(SampleRate / 2.0);
        var bins = new int[FilterCount + 2];
        for (var i = 0; i < bins.Length; i++)
        {
            var mel = low + (high - low) * i / (bins.Length - 1);
            bins[i] = Math.Clamp((int)Math.Floor((FftSize + 1) * MelToHz(mel) / SampleRate), 0, FftSize / 2);
        }
        return bins;
    }

    private static float Similarity(float[] a, float[] b)
    {
        var length = Math.Min(a.Length, b.Length);
        double dot = 0, aa = 0, bb = 0;
        for (var i = 0; i < length; i++) { dot += a[i] * b[i]; aa += a[i] * a[i]; bb += b[i] * b[i]; }
        if (aa <= 0 || bb <= 0) return 0;
        return (float)Math.Clamp((dot / Math.Sqrt(aa * bb) + 1) / 2, 0, 1);
    }

    private static void Normalize(float[] vector)
    {
        var norm = Math.Sqrt(vector.Sum(x => x * x));
        if (norm <= 0) return;
        for (var i = 0; i < vector.Length; i++) vector[i] /= (float)norm;
    }

    public void Dispose() { _session?.Dispose(); _lock.Dispose(); }
}