diff --git a/.gitignore b/.gitignore index f433a9a..40c76b3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,19 @@ +## Logs +src/logs/* + +## Trained Models +src/models/trained_models/separation/* +src/models/trained_models/transcription/* + ## Datasets # Ignore all files in these specific directories -/src/data/processed/test/model1/* -/src/data/processed/test/model2/* -/src/data/processed/train/model1/* -/src/data/processed/train/model2/* -/src/data/raw/model1/* -/src/data/raw/model2/* +src/data/processed/test/model1/* +src/data/processed/test/model2/* +src/data/processed/train/model1/* +src/data/processed/train/model2/* +src/data/raw/model1/* +src/data/raw/model2/* # Except the directory structures !/src/data/processed/test/model1/.gitkeep diff --git a/src/config/constants.py b/src/config/constants.py index e091910..467ae2b 100644 --- a/src/config/constants.py +++ b/src/config/constants.py @@ -8,7 +8,7 @@ # ---------- # Model 1 Constants -MODEL1_SUBSET = "V2" +MODEL1_SUBSET = "V5" MODEL1_BASE_PATH = f"../data/raw/model1/{MODEL1_SUBSET}" MODEL1_TRAIN_FOLDER_PATH = "../data/processed/train/model1" MODEL1_TRAIN_FILE_NAME = f"mix_bass_train_data_{MODEL1_SUBSET}-TRAIN" @@ -38,7 +38,11 @@ TRAINED_MODEL2_SAVE_PATH = "../models/trained_models/transcription/mark1.pt" # Visualisation and Testing constants -TRAINED_AUDIO_FILE_PATH = "../visualization/audio/trained_audio" -PRED_AUDIO_FILE_PATH = "../visualization/audio/predicted_audio" -VISUALIZATION_SAVE_PATH = "../visualization/spectrograms/outputs" +TRAINED_AUDIO_FILE_PATH = ( + "../../../../Bachelor Thesis/thesis/audio outputs/trained audio" +) +PRED_AUDIO_FILE_PATH = ( + "../../../../Bachelor Thesis/thesis/audio outputs/predicted audio" +) +VISUALIZATION_SAVE_PATH = "../../../../Bachelor Thesis/thesis/spectrograms" PRED_MIDI_FILE_PATH = "../visualization/midi/predicted_midi" diff --git a/src/config/fourierparameters.json b/src/config/fourierparameters.json index 2434d58..465546d 100644 --- a/src/config/fourierparameters.json +++ b/src/config/fourierparameters.json @@ -1,8 +1,8 @@ { "n_fft": 2048, "n_mels": 128, - "hop_length": 3308, + "hop_length": 512, "sample_rate": 44100, "track_seconds_considered": 30, - "audio_amplitude_threshold": 0.0001 + "audio_amplitude_threshold": 0.0000001 } \ No newline at end of file diff --git a/src/config/hyperparameters_separation.json b/src/config/hyperparameters_separation.json index 0035fca..b29b499 100644 --- a/src/config/hyperparameters_separation.json +++ b/src/config/hyperparameters_separation.json @@ -1,7 +1,7 @@ { "learning_rate": 0.0001, - "n_epochs": 10, - "hidden_dim": 512, + "n_epochs": 250, + "hidden_dim": 1024, "n_layers": 3, - "dropout_rate": 0.3 + "dropout_rate": 0.1 } \ No newline at end of file diff --git a/src/data_manipulation/__helpers__/normalization/decibel_normalizer.py b/src/data_manipulation/__helpers__/normalization/decibel_normalizer.py new file mode 100644 index 0000000..a5a4484 --- /dev/null +++ b/src/data_manipulation/__helpers__/normalization/decibel_normalizer.py @@ -0,0 +1,38 @@ +import numpy as np + + +class DecibelNormalizer: + def __init__(self, array): + self.array = array + self.min_val = None + self.max_val = None + + def normalize(self): + # Convert spectrogram magnitudes to decibels + epsilon = 1e-10 # small constant to avoid log(0) + self.array = 20 * np.log10(np.maximum(self.array, epsilon)) + + # Find min and max values in the dB-scaled array + self.min_val, self.max_val = np.min(self.array), np.max(self.array) + + # Avoid division-by-zero error in case of all same elements + if self.min_val == self.max_val: + self.array[:] = 0 + else: + self.array = (self.array - self.min_val) / (self.max_val - self.min_val) + + return self.array + + def get_min_max(self): + return self.min_val, self.max_val + + def denormalize(self, min_val, max_val): + if min_val is not None and max_val is not None: + + # Apply the inverse of min-max normalization + self.array = (self.array * (max_val - min_val)) + min_val + + # Convert decibels back to linear scale + self.array = 10 ** (self.array / 20.0) + + return self.array diff --git a/src/data_manipulation/for_production/production_audio_to_dict.py b/src/data_manipulation/for_production/production_audio_to_dict.py index 12a4829..b2bfb7c 100644 --- a/src/data_manipulation/for_production/production_audio_to_dict.py +++ b/src/data_manipulation/for_production/production_audio_to_dict.py @@ -3,8 +3,8 @@ convert_t_dict_key_to_numpy_arrays, convert_to_recarray, ) -from src.data_manipulation.__helpers__.normalization.mix_bass_data_normalizer import ( - Normalizer, +from src.data_manipulation.__helpers__.normalization.decibel_normalizer import ( + DecibelNormalizer, ) from src.transformers.audio_to_freq_time_analysis import audio_to_freq_time_analysis @@ -36,8 +36,8 @@ def production_audio_to_dict(file_name, input_path): t_dict = convert_t_dict_key_to_numpy_arrays(dictionary=t_dict, keys=["x", "y"]) # Normalize the data - norm_x = Normalizer(t_dict["x"]) - t_dict["x"], t_dict["min_max_amplitudes"] = ( + norm_x = DecibelNormalizer(t_dict["x"]) + t_dict["x"], t_dict["x_min_max_amplitudes"] = ( norm_x.normalize(), norm_x.get_min_max(), ) diff --git a/src/data_manipulation/for_training/audio_spectrograms/mixed_signal_to_dict.py b/src/data_manipulation/for_training/audio_spectrograms/mixed_signal_to_dict.py index f94c024..0c006df 100644 --- a/src/data_manipulation/for_training/audio_spectrograms/mixed_signal_to_dict.py +++ b/src/data_manipulation/for_training/audio_spectrograms/mixed_signal_to_dict.py @@ -6,8 +6,8 @@ savez_numpy_data, convert_to_recarray, ) -from src.data_manipulation.__helpers__.normalization.mix_bass_data_normalizer import ( - Normalizer, +from src.data_manipulation.__helpers__.normalization.decibel_normalizer import ( + DecibelNormalizer, ) from src.data_manipulation.__helpers__.truncator.mix_bass_data_truncator import ( data_truncator, @@ -111,16 +111,12 @@ def mixed_signal_to_dict(base_path, files_to_transform, save_file_path, pause=Fa t_dict = convert_t_dict_key_to_numpy_arrays(dictionary=t_dict, keys=["x", "y"]) # Normalize the data - norm_x = Normalizer(t_dict["x"]) - t_dict["x"], t_dict["min_max_amplitudes"] = ( - norm_x.normalize(), - norm_x.get_min_max(), - ) - norm_y = Normalizer(t_dict["y"]) - t_dict["y"], t_dict["min_max_amplitudes"] = ( - norm_y.normalize(), - norm_y.get_min_max(), - ) + norm_x = DecibelNormalizer(t_dict["x"]) + t_dict["x"] = norm_x.normalize() + t_dict["x_min_max_amplitudes"] = norm_x.get_min_max() + norm_y = DecibelNormalizer(t_dict["y"]) + t_dict["y"] = norm_y.normalize() + t_dict["y_min_max_amplitudes"] = norm_y.get_min_max() # Transform to recarray t_dict_recarray = convert_to_recarray(data_dict=t_dict) diff --git a/src/data_manipulation/for_training/tab_transcriptions/bass_and_midi_to_dict.py b/src/data_manipulation/for_training/tab_transcriptions/bass_and_midi_to_dict.py index 7dc95c8..becea1f 100644 --- a/src/data_manipulation/for_training/tab_transcriptions/bass_and_midi_to_dict.py +++ b/src/data_manipulation/for_training/tab_transcriptions/bass_and_midi_to_dict.py @@ -5,8 +5,8 @@ convert_to_recarray, savez_numpy_data, ) -from src.data_manipulation.__helpers__.normalization.mix_bass_data_normalizer import ( - Normalizer, +from src.data_manipulation.__helpers__.normalization.decibel_normalizer import ( + DecibelNormalizer, ) from src.data_manipulation.__helpers__.truncator.mix_bass_data_truncator import ( data_truncator, @@ -95,8 +95,8 @@ def bass_and_midi_to_dict(base_path, files_to_transform, save_file_path, pause=F t_dict = convert_t_dict_key_to_numpy_arrays(dictionary=t_dict, keys=["x", "y"]) # Normalize the data - norm_x = Normalizer(t_dict["x"]) - t_dict["x"], t_dict["min_max_amplitudes"] = ( + norm_x = DecibelNormalizer(t_dict["x"]) + t_dict["x"], t_dict["x_min_max_amplitudes"] = ( norm_x.normalize(), norm_x.get_min_max(), ) diff --git a/src/main.py b/src/main.py index ff99f79..415f4ec 100644 --- a/src/main.py +++ b/src/main.py @@ -15,80 +15,11 @@ def main(): - while True: - choice = ( - input( - "@@@@@@ Would you like to create new models before processing your audio? @@@@@@\n" - '@@@@@@ WARNING: Selecting "yes" requires training/testing datasets to be present in the ' - "data/raw/model1 and data/raw/model2 folders. @@@@@@\n" - '@@@@@@ Selecting "no" will use the pre-existing mark1 models instead. [Y/N]:' - ) - .strip() - .lower() - ) - if choice in ["yes", "y"]: - # Trains and tests both models separately - print() - print("@@@@@@ SEPARATION TRAINING/TESTING START @@@@@@") - separation_manager() - print() - print("@@@@@@ TRANSCRIPTION TRAINING/TESTING START @@@@@@") - transcription_manager() - print() - print("@@@@@@ MODELS SUCCESSFULLY CREATED @@@@@@") - break - - elif choice in ["no", "n"]: - break - - else: - print("Please enter a valid input. Either [Y/N] or [Yes/No].") - - try: - input_folder = f"{PATH_TO_AUDIO}/{PRODUCTION_INPUT_FOLDER_PATH}" - if not os.listdir(input_folder): - print("No input files present in input folder. Programm terminated.") - return - for track in os.listdir(f"{input_folder}"): - if track.endswith(".wav"): - print() - print(f"@@@@ USING MODELS ON SELECTED TRACK: {track} @@@@") - input_file_path = ( - f"{PATH_TO_AUDIO}/{PRODUCTION_INPUT_FOLDER_PATH}/{track}" - ) - output_file_path = ( - f"{PATH_TO_AUDIO}/{PRODUCTION_OUTPUT_FOLDER_PATH}/{track}" - ) - - print("@@ Pre-Processing... @@") - # Pre-process the audio into correct spectrogram format - processed_input = transform_data( - flag="production input", - file_name=track, - input_file_path=input_file_path, - ) - - print("@@ Separating and transcribing... @@") - # Pass processed audio through both models - output = use_models_on_audio(processed_input) - - print("@@ Post-Processing... @@") - # Post-process the midi-spectrogram into pure midi format and save it - piano_roll_to_midi( - piano_roll=output, - output_file_path=output_file_path, - tag="", - mix_names=["mix_name"], - flag="production", - ) - - else: - print(f"@@@@ {track} is not a .wav file. Skipping... @@@@") - except Exception as e: - raise Exception( - f"An error occurred while processing the production audio files: {e}" - ) + print("@@@@@@ SEPARATION TRAINING/TESTING START @@@@@@") + separation_manager() + print() + print("@@@@@@ MODEL SUCCESSFULLY CREATED @@@@@@") if __name__ == "__main__": diff --git a/src/models/__helpers__/visualize_for_evaluation.py b/src/models/__helpers__/visualize_for_evaluation.py deleted file mode 100644 index 54903c3..0000000 --- a/src/models/__helpers__/visualize_for_evaluation.py +++ /dev/null @@ -1,27 +0,0 @@ -from src.config.constants import VISUALIZATION_SAVE_PATH, TRAINED_AUDIO_FILE_PATH -from src.transformers.freq_time_analysis_to_audio import freq_time_analysis_to_audio -from src.visualization.spectrograms.visualize_spectrograms import visualize_spectrograms - - -def visualize_for_evaluation(outputs, x, y, data, flag, tag): - # Convert tensor back into numpy array - outputs_for_visualization = outputs.detach().cpu().numpy() - # Convert first three tracks back to audio for review - x_train_for_visualization = x.detach().cpu().numpy() - y_train_for_visualization = y.detach().cpu().numpy() - visualize_spectrograms( - VISUALIZATION_SAVE_PATH, - x_train_for_visualization[0], - y_train_for_visualization[0], - outputs_for_visualization[0], - data["mix_name"], - tag=tag, - ) - if flag: - freq_time_analysis_to_audio( - mel_spectrogram_array=outputs_for_visualization[:3], - output_file_path=TRAINED_AUDIO_FILE_PATH, - mix_names=data["mix_name"], - min_max_amplitudes=data["min_max_amplitudes"], - tag=tag, - ) diff --git a/src/models/__helpers__/visualize_for_testing_evaluation.py b/src/models/__helpers__/visualize_for_testing_evaluation.py new file mode 100644 index 0000000..af886b8 --- /dev/null +++ b/src/models/__helpers__/visualize_for_testing_evaluation.py @@ -0,0 +1,31 @@ +from src.config.constants import VISUALIZATION_SAVE_PATH +from src.visualization.spectrograms.visualize_spectrogram import visualize_spectrogram + + +def visualize_for_testing_evaluation(y_pred, y_test, x_test, data, tag): + # Convert tensor back into numpy array + y_pred_for_visualization = y_pred.detach().cpu().numpy() + x_test_for_visualization = x_test.detach().cpu().numpy() + y_test_for_visualization = y_test.detach().cpu().numpy() + + visualize_spectrogram( + save_folder_path=VISUALIZATION_SAVE_PATH, + mel_spectrogram=x_test_for_visualization[0], + mix_name=data["mix_name"], + tag=tag, + label="x_test", + ) + visualize_spectrogram( + save_folder_path=VISUALIZATION_SAVE_PATH, + mel_spectrogram=y_pred_for_visualization[0], + mix_name=data["mix_name"], + tag=tag, + label="y_pred", + ) + visualize_spectrogram( + save_folder_path=VISUALIZATION_SAVE_PATH, + mel_spectrogram=y_test_for_visualization[0], + mix_name=data["mix_name"], + tag=tag, + label="y_test", + ) diff --git a/src/models/__helpers__/visualize_for_training_evaluation.py b/src/models/__helpers__/visualize_for_training_evaluation.py new file mode 100644 index 0000000..081bd35 --- /dev/null +++ b/src/models/__helpers__/visualize_for_training_evaluation.py @@ -0,0 +1,41 @@ +from src.config.constants import VISUALIZATION_SAVE_PATH, TRAINED_AUDIO_FILE_PATH +from src.transformers.freq_time_analysis_to_audio import freq_time_analysis_to_audio +from src.visualization.spectrograms.visualize_spectrogram import visualize_spectrogram + + +def visualize_for_training_evaluation(outputs, x, y, data, flag, tag): + # Convert tensor back into numpy array + outputs_for_visualization = outputs.detach().cpu().numpy() + x_train_for_visualization = x.detach().cpu().numpy() + y_train_for_visualization = y.detach().cpu().numpy() + + visualize_spectrogram( + save_folder_path=VISUALIZATION_SAVE_PATH, + mel_spectrogram=x_train_for_visualization[0], + mix_name=data["mix_name"], + tag=tag, + label="x_train", + ) + visualize_spectrogram( + save_folder_path=VISUALIZATION_SAVE_PATH, + mel_spectrogram=y_train_for_visualization[0], + mix_name=data["mix_name"], + tag=tag, + label="y_train", + ) + visualize_spectrogram( + save_folder_path=VISUALIZATION_SAVE_PATH, + mel_spectrogram=outputs_for_visualization[0], + mix_name=data["mix_name"], + tag=tag, + label="y_output", + ) + + if flag: + freq_time_analysis_to_audio( + mel_spectrogram_array=outputs_for_visualization[:3], + output_file_path=TRAINED_AUDIO_FILE_PATH, + mix_names=data["mix_name"], + min_max_amplitudes=data["y_min_max_amplitudes"], + tag=tag, + ) diff --git a/src/models/audio_separation/gru/gru_separation.py b/src/models/audio_separation/gru/gru_separation.py index 0c11b8f..4212252 100644 --- a/src/models/audio_separation/gru/gru_separation.py +++ b/src/models/audio_separation/gru/gru_separation.py @@ -15,25 +15,29 @@ def __init__(self, input_size, hidden_dim, n_layers, output_size, dropout_rate): # Dropout rate self.dropout_rate = dropout_rate - # GRU - self.gru = nn.GRU(input_size, hidden_dim, n_layers, batch_first=True) + # Bi-Directional GRU + self.gru = nn.GRU( + input_size, hidden_dim, n_layers, batch_first=True, bidirectional=True + ) # Dropout layer self.dropout = nn.Dropout(dropout_rate) # Readout layers - self.fc1 = nn.Linear(hidden_dim, hidden_dim * 2) - self.fc2 = nn.Linear(hidden_dim * 2, hidden_dim * 4) - self.fc3 = nn.Linear(hidden_dim * 4, output_size) + self.fc1 = nn.Linear(self.hidden_dim * 2, self.hidden_dim * 2) + self.layer_norm1 = nn.LayerNorm(self.hidden_dim * 2) + self.fc2 = nn.Linear(self.hidden_dim * 2, self.hidden_dim * 4) + self.layer_norm2 = nn.LayerNorm(self.hidden_dim * 4) + self.fc3 = nn.Linear(self.hidden_dim * 4, output_size) + + # Sigmoid activation for output bounding between 0 and 1. + self.sigmoid = nn.Sigmoid() def forward(self, x): """ `x` is the batch of sequences that you want your RNN to process. - `x` has a shape of `(batch_size, seq_length, num_features)`: - - `batch_size` is the number of sequences you process at a time. - - `seq_length` is the length of each sequence. - - `num_features` is the number of input features at each sequence element. + `x` has a shape of `(batch_size, freq_bins, time_steps)` """ batch_size = x.size(0) @@ -47,15 +51,22 @@ def forward(self, x): out = self.dropout(out) out = self.fc1(out) - out = torch.relu(out) + out = torch.relu(self.layer_norm1(out.view(-1, out.size(2)))) + out = out.view(batch_size, -1, self.hidden_dim * 2) + out = self.fc2(out) - out = torch.relu(out) + out = torch.relu(self.layer_norm2(out.view(-1, out.size(2)))) + out = out.view(batch_size, -1, self.hidden_dim * 4) + out = self.fc3(out) + # Apply sigmoid activation to bound the outputs between 0 and 1 + out = self.sigmoid(out) + return out, hidden_j def init_hidden(self, batch_size): # Generates the first hidden state of zeros for the forward pass - hidden = torch.zeros(self.n_layers, batch_size, self.hidden_dim) + hidden = torch.zeros(self.n_layers * 2, batch_size, self.hidden_dim) return hidden diff --git a/src/models/audio_separation/rnn/rnn.py b/src/models/audio_separation/rnn/rnn.py index e9e1b4b..c260a34 100644 --- a/src/models/audio_separation/rnn/rnn.py +++ b/src/models/audio_separation/rnn/rnn.py @@ -32,10 +32,7 @@ def forward(self, x): """ `x` is the batch of sequences that you want your RNN to process. - `x` has a shape of `(batch_size, seq_length, num_features)`: - - `batch_size` is the number of sequences you process at a time. - - `seq_length` is the length of each sequence. - - `num_features` is the number of input features at each sequence element. + `x` has a shape of `(batch_size, freq_bins, time_steps)` """ batch_size = x.size(0) diff --git a/src/models/audio_separation/separation_manager.py b/src/models/audio_separation/separation_manager.py index 1728a9d..b2f07e6 100644 --- a/src/models/audio_separation/separation_manager.py +++ b/src/models/audio_separation/separation_manager.py @@ -16,6 +16,7 @@ data_truncator, ) from src.models.audio_separation.gru.gru_separation import GRU_Separation +from src.models.audio_separation.rnn.rnn import RNN from src.models.test import test from src.models.train import train from src.transformers.freq_time_analysis_to_audio import freq_time_analysis_to_audio @@ -30,8 +31,28 @@ def separation_manager(): - # Transform training/testing data for audio separation - transform_data(flag="audio separation") + while True: + choice = ( + input( + "@@@@@@ Would you like to pre-process the data [1] or use the existing pre-processed data [2]?:" + ) + .strip() + .lower() + ) + + if choice in ["1"]: + # Transform training/testing data for audio separation + print() + print("@@@@@@ DATA PRE-PROCESSING START @@@@@@") + transform_data(flag="audio separation") + + break + + elif choice in ["2"]: + break + + else: + print("Please enter a valid input. Either [1] or [2].") # Load the training dataset print("@@@@ Loading the training dataset @@@@") @@ -76,7 +97,7 @@ def separation_manager(): y_test = y_test.float() # Initialize the model - model = GRU_Separation( + model = RNN( input_size=x_train.shape[2], hidden_dim=hyperparameters["hidden_dim"], n_layers=hyperparameters["n_layers"], @@ -100,13 +121,13 @@ def separation_manager(): torch.save(model.state_dict(), TRAINED_MODEL1_SAVE_PATH) # Test the model - y_pred = test(x_test, y_test, model, criterion) + y_pred = test(x_test, y_test, model, criterion, data_test, tag="separation") # Convert first three tracks back to audio for review freq_time_analysis_to_audio( mel_spectrogram_array=y_pred[:3], output_file_path=PRED_AUDIO_FILE_PATH, mix_names=data_test["mix_name"], - min_max_amplitudes=data_test["min_max_amplitudes"], + min_max_amplitudes=data_test["y_min_max_amplitudes"], tag="SEPARATION-TESTING", ) diff --git a/src/models/test.py b/src/models/test.py index a213181..d9271af 100644 --- a/src/models/test.py +++ b/src/models/test.py @@ -1,4 +1,9 @@ -def test(x_test, y_test, model, criterion): +from src.models.__helpers__.visualize_for_testing_evaluation import ( + visualize_for_testing_evaluation, +) + + +def test(x_test, y_test, model, criterion, data, tag): print("@@@@ Starting model testing @@@@") # Switch the model to evaluation mode to turn off features like dropout model.eval() @@ -9,6 +14,8 @@ def test(x_test, y_test, model, criterion): test_loss = criterion(y_pred, y_test) print(f"@@@@ Test loss: {test_loss.item():.7f} @@@@") + visualize_for_testing_evaluation(y_pred, y_test, x_test, data, tag) + # Convert tensor back into numpy array y_pred = y_pred.detach().cpu().numpy() diff --git a/src/models/train.py b/src/models/train.py index 77b7f89..8405825 100644 --- a/src/models/train.py +++ b/src/models/train.py @@ -1,7 +1,12 @@ import json import os +import torch.optim.lr_scheduler as lr_scheduler +from torch.utils.tensorboard import SummaryWriter +import torch.nn.utils.clip_grad from src.models.__helpers__.learning_rate_reducer import learning_rate_reducer -from src.models.__helpers__.visualize_for_evaluation import visualize_for_evaluation +from src.models.__helpers__.visualize_for_training_evaluation import ( + visualize_for_training_evaluation, +) dir_path = os.path.dirname(os.path.realpath(__file__)) hyperparameters_separation_path = os.path.join( @@ -17,6 +22,9 @@ def train(x_train, y_train, model, criterion, optimizer, data_train, tag): + # TensorBoard Logging + writer = SummaryWriter(log_dir=os.path.join(dir_path, "../logs/tensorboard_logs/")) + if tag == "separation": hyperparameters = hyperparameters_separation elif tag == "transcription": @@ -24,8 +32,14 @@ def train(x_train, y_train, model, criterion, optimizer, data_train, tag): else: raise Exception("Incorrect tag") - # Track loss to break training loop if loss is no longer changing + # Create the lr scheduler + scheduler = lr_scheduler.ReduceLROnPlateau( + optimizer, "min", patience=5, factor=0.5, verbose=True + ) + + # Track loss to break training loop if loss is no longer changing or increasing no_change = 0 + loss_increasing = False prev_loss = float("inf") # Track learning rate reduction @@ -52,18 +66,31 @@ def train(x_train, y_train, model, criterion, optimizer, data_train, tag): # Backward pass (backpropagation) where gradients are calculated loss.backward() + + # Gradient clipping + torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=0.5) + # Update model parameters, based on the gradients calculated in the backward pass optimizer.step() + # TensorBoard Logging + writer.add_scalar("Loss/train", loss.item(), epoch) + + # Log parameter and gradient histograms + for name, param in model.named_parameters(): + writer.add_histogram(f"Params/{name}", param, epoch) + if param.grad is not None: + writer.add_histogram(f"Grads/{name}", param.grad, epoch) + # Print statistics print(f"@@ Epoch {epoch + 1} Done. loss: {loss.item():.7f} @@") - # Reduce learning rate if necessary - lr_reduced, optimizer = learning_rate_reducer(loss, optimizer, lr_reduced) + # Update the scheduler with the current loss + scheduler.step(loss.item()) # Visualizations and audio-transforms for manual evaluation if epoch == hyperparameters["n_epochs"] - 1: - visualize_for_evaluation( + visualize_for_training_evaluation( outputs, x_train, y_train, @@ -78,19 +105,42 @@ def train(x_train, y_train, model, criterion, optimizer, data_train, tag): else: no_change = 0 + # Check if the loss over the last 5 epochs has been larger than its previous loss + if loss.item() > prev_loss: + loss_increasing += 1 + else: + loss_increasing = 0 + prev_loss = loss.item() + # If loss has started increasing, stop early + if loss_increasing >= 10: + print( + "@@@@@@ Stopping early - loss has increased too many times in a row. @@@@@@ " + ) + visualize_for_training_evaluation( + outputs=outputs, + x=x_train, + y=y_train, + data=data_train, + tag=f"{tag}-TRAINING", + flag=True, + ) + break + # If loss hasn't changed for 30 epochs, stop early if no_change >= 30: print("@@@@@@ Stopping early - loss hasn't changed in 30 epochs. @@@@@@ ") - visualize_for_evaluation( + visualize_for_training_evaluation( outputs, x_train, y_train, data_train, tag=f"{tag}-TRAINING", - flag=False, + flag=True, ) break + writer.close() + return model diff --git a/src/models/trained_models/separation/mark1.pt b/src/models/trained_models/separation/mark1.pt deleted file mode 100644 index c85617a..0000000 Binary files a/src/models/trained_models/separation/mark1.pt and /dev/null differ diff --git a/src/models/trained_models/transcription/mark1.pt b/src/models/trained_models/transcription/mark1.pt deleted file mode 100644 index f1137d7..0000000 Binary files a/src/models/trained_models/transcription/mark1.pt and /dev/null differ diff --git a/src/models/use_models_on_audio.py b/src/models/use_models_on_audio.py index a70b740..d5765f0 100644 --- a/src/models/use_models_on_audio.py +++ b/src/models/use_models_on_audio.py @@ -4,6 +4,7 @@ from src.config.constants import SEPARATION_MODEL_PATH, TRANSCRIPTION_MODEL_PATH from src.models.audio_separation.gru.gru_separation import GRU_Separation +from src.models.audio_separation.rnn.rnn import RNN from src.models.tab_transcription.gru.gru_transcription import GRU_Transcription dir_path = os.path.dirname(os.path.realpath(__file__)) @@ -29,7 +30,7 @@ def use_models_on_audio(input_data): y_separation = y_separation.float() # Load trained separation model - separation_model = GRU_Separation( + separation_model = RNN( input_size=x_separation.shape[2], hidden_dim=hyperparameters_separation["hidden_dim"], n_layers=hyperparameters_separation["n_layers"], diff --git a/src/scripts/rename_files.py b/src/scripts/rename_files.py new file mode 100644 index 0000000..3d3c72e --- /dev/null +++ b/src/scripts/rename_files.py @@ -0,0 +1,35 @@ +import os + + +path = "../data/raw/model1/V2" + + +folders = os.listdir(path) + +counter = 1 +# Iterate over each folder +for folder in folders: + folder_path = os.path.join(path, folder) + # Rename mixture.wav file + mixture_file_path = os.path.join(folder_path, "mixture.wav") + if os.path.exists(mixture_file_path): + new_mixture_file_name = f"Track{counter}.wav" + new_mixture_file_path = os.path.join(folder_path, new_mixture_file_name) + + # Rename the mixture.wav file + os.rename(mixture_file_path, new_mixture_file_path) + print(f"Renamed {mixture_file_path} to {new_mixture_file_path}") + + # Rename Bass/Bass.wav file + bass_folder_path = os.path.join(folder_path, "Bass") + bass_file_path = os.path.join(bass_folder_path, "bass.wav") + if os.path.exists(bass_file_path): + new_bass_file_name = f"Bass{counter}.wav" + new_bass_file_path = os.path.join(bass_folder_path, new_bass_file_name) + + # Rename the Bass.wav file + os.rename(bass_file_path, new_bass_file_path) + print(f"Renamed {bass_file_path} to {new_bass_file_path}") + + # Increment the counter + counter += 1 diff --git a/src/transformers/audio_to_freq_time_analysis.py b/src/transformers/audio_to_freq_time_analysis.py index 1de095c..3024c11 100644 --- a/src/transformers/audio_to_freq_time_analysis.py +++ b/src/transformers/audio_to_freq_time_analysis.py @@ -44,9 +44,12 @@ def audio_to_freq_time_analysis(file_path, flag=False): hop_length = fourierparameters["hop_length"] n_fft = fourierparameters["n_fft"] n_mels = fourierparameters["n_mels"] + win_length = fourierparameters["n_fft"] # Perform stft - stft = librosa.stft(signal, n_fft=n_fft, hop_length=hop_length) + stft = librosa.stft( + signal, n_fft=n_fft, hop_length=hop_length, win_length=win_length + ) # Calculate abs values on complex numbers to get magnitude mag_spectrogram, phase = librosa.magphase(stft) diff --git a/src/transformers/freq_time_analysis_to_audio.py b/src/transformers/freq_time_analysis_to_audio.py index 7c94898..5f2d534 100644 --- a/src/transformers/freq_time_analysis_to_audio.py +++ b/src/transformers/freq_time_analysis_to_audio.py @@ -3,8 +3,8 @@ import json import os -from src.data_manipulation.__helpers__.normalization.mix_bass_data_normalizer import ( - Normalizer, +from src.data_manipulation.__helpers__.normalization.decibel_normalizer import ( + DecibelNormalizer, ) from scipy.signal import butter, lfilter @@ -40,10 +40,12 @@ def freq_time_analysis_to_audio( for track in range(mel_spectrogram_array.shape[0]): track_counter += 1 min_val, max_val = min_max_amplitudes - mel_spectrogram_array[track] = Normalizer( + + mel_spectrogram_array[track] = DecibelNormalizer( mel_spectrogram_array[track] ).denormalize(min_val, max_val) - print(f"@@@@ Recreating audio of track {track} @@@@") + + print(f"@@@@ Recreating audio of track {mix_names[track]} @@@@") spectrogram_array = librosa.feature.inverse.mel_to_stft( mel_spectrogram_array, sr=fourierparameters["sample_rate"], @@ -57,10 +59,11 @@ def freq_time_analysis_to_audio( hop_length=fourierparameters["hop_length"], win_length=fourierparameters["n_fft"], n_fft=fourierparameters["n_fft"], + n_iter=128, ) audio = lowpass_filter( - audio, cutoff=1500, sr=fourierparameters["sample_rate"] + audio, cutoff=2500, sr=fourierparameters["sample_rate"] ) # Save the audio to file diff --git a/src/visualization/fourier_and_back_trial/fourier_audio_loss_trials.py b/src/visualization/fourier_and_back_trial/fourier_audio_loss_trials.py index 05407b3..76bea4f 100644 --- a/src/visualization/fourier_and_back_trial/fourier_audio_loss_trials.py +++ b/src/visualization/fourier_and_back_trial/fourier_audio_loss_trials.py @@ -1,7 +1,7 @@ import json import os -from src.data_manipulation.__helpers__.normalization.mix_bass_data_normalizer import ( - Normalizer, +from src.data_manipulation.__helpers__.normalization.decibel_normalizer import ( + DecibelNormalizer, ) from src.transformers.audio_to_freq_time_analysis import audio_to_freq_time_analysis from src.transformers.freq_time_analysis_to_audio import freq_time_analysis_to_audio @@ -11,7 +11,7 @@ with open("../../config/fourierparameters.json") as fourierparameters_file: fourierparameters = json.load(fourierparameters_file) -INPUT_FILE_PATH = "audio/MusicDelta_80sRock/Bass/MusicDelta_80sRock_STEM_02.wav" +INPUT_FILE_PATH = "audio/Track56.wav" def fourier_audio_loss(file_path): @@ -21,12 +21,12 @@ def fourier_audio_loss(file_path): """ try: mel_spectrogram, phase = audio_to_freq_time_analysis(file_path) - t_dict = {"x": list(), "phase": list(), "min_max_amplitudes": list()} + t_dict = {"x": list(), "phase": list(), "x_min_max_amplitudes": list()} t_dict["x"].append(mel_spectrogram) - norm_x = Normalizer(t_dict["x"]) - t_dict["x"], t_dict["min_max_amplitudes"] = ( + norm_x = DecibelNormalizer(t_dict["x"]) + t_dict["x"], t_dict["x_min_max_amplitudes"] = ( norm_x.normalize(), norm_x.get_min_max(), ) @@ -34,11 +34,14 @@ def fourier_audio_loss(file_path): freq_time_analysis_to_audio( mel_spectrogram_array=t_dict["x"], - output_file_path="audio/", - mix_names=["MusicDelta_80sRock_MIX.wav"], - min_max_amplitudes=t_dict["min_max_amplitudes"], - tag="TRIAL-", + output_file_path="audio", + mix_names=["Track56.wav"], + min_max_amplitudes=t_dict["x_min_max_amplitudes"], + tag="TRIAL", ) except Exception as e: raise Exception("Exception occurred: {}".format(e)) + + +fourier_audio_loss(INPUT_FILE_PATH) diff --git a/src/visualization/spectrograms/visualize_spectrogram.py b/src/visualization/spectrograms/visualize_spectrogram.py new file mode 100644 index 0000000..ac8b584 --- /dev/null +++ b/src/visualization/spectrograms/visualize_spectrogram.py @@ -0,0 +1,49 @@ +import numpy as np +import matplotlib.pyplot as plt +import librosa +import librosa.display +import os +import json + + +dir_path = os.path.dirname(os.path.realpath(__file__)) + +fourierparameters_path = os.path.join(dir_path, "../../config/fourierparameters.json") + +with open(fourierparameters_path) as fourierparameters_file: + fourierparameters = json.load(fourierparameters_file) + + +def visualize_spectrogram( + save_folder_path, + mel_spectrogram, + mix_name, + tag, + label, +): + """ + Use this function to create graphs that help recognize the success of the separation model. + """ + + sr = fourierparameters["sample_rate"] + hop_length = fourierparameters["hop_length"] + + print(f"@@@@ Creating {label} spectrogram @@@@") + plt.figure(figsize=(10, 4)) + db_spectrogram = librosa.amplitude_to_db( + mel_spectrogram, ref=np.max(mel_spectrogram) + ) + librosa.display.specshow( + db_spectrogram, + y_axis="mel", + fmax=8000, + x_axis="s", + sr=sr, + hop_length=hop_length, + vmin=np.min(db_spectrogram), + vmax=np.max(db_spectrogram), + ) + plt.colorbar(format="%+2.0f dB") + plt.title(f"Mel spectrogram - {label} - {mix_name[0]}") + plt.tight_layout() + plt.savefig(f"{save_folder_path}/{tag}-spectrogram-{label}.png") diff --git a/src/visualization/spectrograms/visualize_spectrograms.py b/src/visualization/spectrograms/visualize_spectrograms.py deleted file mode 100644 index 5bbdcb7..0000000 --- a/src/visualization/spectrograms/visualize_spectrograms.py +++ /dev/null @@ -1,75 +0,0 @@ -import numpy as np -import matplotlib.pyplot as plt -import librosa -import librosa.display -import os -import json - - -dir_path = os.path.dirname(os.path.realpath(__file__)) - -fourierparameters_path = os.path.join(dir_path, "../../config/fourierparameters.json") - -with open(fourierparameters_path) as fourierparameters_file: - fourierparameters = json.load(fourierparameters_file) - - -def visualize_spectrograms( - save_folder_path, - mel_spectrogram_x_train, - mel_spectrogram_y_train, - mel_spectrogram_y_train_output, - mix_name, - tag, -): - """ - Use this function to create graphs that help recognize the training success of the separation model. - """ - - sr = fourierparameters["sample_rate"] - hop_length = fourierparameters["hop_length"] - - print("@@@@ Creating x_train spectrogram @@@@") - plt.figure(figsize=(10, 4)) - librosa.display.specshow( - librosa.power_to_db(mel_spectrogram_x_train, ref=np.max), - y_axis="mel", - fmax=8000, - x_axis="time", - sr=sr, - hop_length=hop_length, - ) - plt.colorbar(format="%+2.0f dB") - plt.title(f"Mel spectrogram - x_train - {mix_name[0]}") - plt.tight_layout() - plt.savefig(f"{save_folder_path}/{tag}-spectrogram-x_train.png") - - print("@@@@ Creating y_train spectrogram @@@@") - plt.figure(figsize=(10, 4)) - librosa.display.specshow( - librosa.power_to_db(mel_spectrogram_y_train, ref=np.max), - y_axis="mel", - fmax=8000, - x_axis="time", - sr=sr, - hop_length=hop_length, - ) - plt.colorbar(format="%+2.0f dB") - plt.title(f"Mel spectrogram - {mix_name[0]}") - plt.tight_layout() - plt.savefig(f"{save_folder_path}/{tag}-spectrogram-y_train.png") - - print("@@@@ Creating y_train_output spectrogram @@@@") - plt.figure(figsize=(10, 4)) - librosa.display.specshow( - librosa.power_to_db(mel_spectrogram_y_train_output, ref=np.max), - y_axis="mel", - fmax=8000, - x_axis="time", - sr=sr, - hop_length=hop_length, - ) - plt.colorbar(format="%+2.0f dB") - plt.title(f"Mel spectrogram - {mix_name[0]}") - plt.tight_layout() - plt.savefig(f"{save_folder_path}/{tag}-spectrogram-y_train_output.png")