soundtypes/direct_matching.py at master · CarmineCella/soundtypes · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
# -*- coding: utf-8 -*-
"""
@author: Carmine-Emanuele Cella, 2016
"""

import numpy as np
import librosa
import matplotlib.pyplot as plt
from sklearn.neighbors import NearestNeighbors
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
import soundfile as sf

SR = 44100
N_COEFF = 20
K = 5
SOURCE_FILE = 'samples/cage.wav'
TARGET_FILE = 'samples/Kapustin_prelude_cut.wav'
FRAME_SIZE = 2048
HOP_SIZE = 1024
SCALING = 100

if __name__ == "__main__":
    print ('[direct timbre matching]\n')
    print ('computing features...')
    [y_src, sr] = librosa.core.load(SOURCE_FILE, sr=SR)

    y_pad_src = np.zeros(len(y_src) + FRAME_SIZE)
    y_pad_src[1:len(y_src)+1] = y_src

    C_src = librosa.feature.mfcc(y=y_src, sr=sr, n_mfcc=N_COEFF,
                                 n_fft=FRAME_SIZE, hop_length=HOP_SIZE)
    [y_dst, sr] = librosa.core.load(TARGET_FILE, sr= SR)

    y_pad_dst = np.zeros(len(y_dst) + FRAME_SIZE)
    y_pad_dst[1:len(y_dst)+1] = y_dst

    C_dst = librosa.feature.mfcc(y=y_dst, sr=sr, n_mfcc=N_COEFF,
                                 n_fft=FRAME_SIZE, hop_length=HOP_SIZE)

    C_scaled_dst = C_dst.T
    C_scaled_src = C_src.T

    scaler = StandardScaler ()
    C_scaled_dst = scaler.fit_transform (C_scaled_dst)
    C_scaled_src = scaler.fit_transform (C_scaled_src)

    print ('fitting datasets...')
    knn = NearestNeighbors(n_neighbors=K).fit (C_scaled_dst);

    print ('generate hybridization...')

    n_frames = C_scaled_src.shape[0]
    gen_sound = np.zeros(n_frames * HOP_SIZE + FRAME_SIZE)
    for i in range(n_frames):
        dist, idx = knn.kneighbors(C_scaled_src[i, :].reshape(1, -1))
        atom = idx[0][np.random.randint(K)]
        amp = np.sum (np.abs(y_pad_src[i * HOP_SIZE : i * HOP_SIZE + \
            FRAME_SIZE]))
        chunk = y_pad_dst[atom * HOP_SIZE : atom * HOP_SIZE + FRAME_SIZE] \
            * np.hanning(FRAME_SIZE)

        chunk /= SCALING
        gen_sound[i * HOP_SIZE : i * HOP_SIZE + FRAME_SIZE] += (chunk * amp)

    print ('saving audio data...')
    sf.write('generated_sound.wav', gen_sound, sr)


    pca = PCA(2)
    C_scaled_dst = pca.fit_transform (C_scaled_dst)
    C_scaled_src = pca.fit_transform (C_scaled_src)

    plt.close ('all')

    plt.figure ()
    plt.plot (C_scaled_src[:, 0], C_scaled_src[:, 1], 'go')
    plt.plot (C_scaled_dst[:, 0], C_scaled_dst[:, 1], 'ro')
    plt.title('source and destination points')