From 7f0ac81c3ee707eff445d44cff3e53ecce11a044 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kov=C3=A1cs=20G=C3=A1bor?= Date: Wed, 6 Apr 2022 14:35:15 +0200 Subject: [PATCH 1/6] some of the investigation types --- info_nce.py | 12 ++++++++++++ main_linearpredhead.py | 12 +++++++----- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/info_nce.py b/info_nce.py index 136f371..76e948c 100644 --- a/info_nce.py +++ b/info_nce.py @@ -26,6 +26,8 @@ def __init__(self, tau: float = 0.07, grad_scaling=False): def forward(self, z1: torch.Tensor, z2: torch.Tensor) -> torch.Tensor: + run_type = ["label_smoothing", "top_k_based_on_cos_sim"][1] + # Collect from all gpu z1 = AllGather.apply(z1) z2 = AllGather.apply(z2) @@ -41,9 +43,19 @@ def forward(self, z1: torch.Tensor, z2: torch.Tensor) -> torch.Tensor: ones = torch.ones(n // 2).to(z.device) labels = ones.diagflat(n // 2) + ones.diagflat(-n // 2) + if run_type == "label_smoothing": + labels = labels.clamp(min=0.01) + labels = F.normalize(labels, dim=1) + # Note: The following code might require a large amount of memory # in case of large batch size sim_m = z @ z.T + F.normalize(z, dim=1, p=2) + if run_type == "top_k_based_on_cos_sim": + cos_sim_m = F.softmax(z @ z.T, dim=1) + top_k, top_k_indices = torch.topk(cos_sim_m, k=10, dim=1) + top_sim = torch.zeros_like(cos_sim_m).scatter(dim=1, index=top_k_indices, src=top_k) + labels = torch.max(labels, top_sim) # This is a bit of cheat. Instead of removing cells from # the matrix where i==j, instead we set it to a very small value diff --git a/main_linearpredhead.py b/main_linearpredhead.py index e5f5335..ea121de 100755 --- a/main_linearpredhead.py +++ b/main_linearpredhead.py @@ -41,7 +41,7 @@ and callable(models.__dict__[name])) parser = argparse.ArgumentParser(description='PyTorch ImageNet Training') -parser.add_argument("--runname", default="dev", help="Name of run on tensorboard") +parser.add_argument("--runname", default="top_10_based_on_cos_sim", help="Name of run on tensorboard") parser.add_argument('--data', metavar='DIR', help='path to dataset', default='/shared_data/imagenet') parser.add_argument('-a', '--arch', metavar='ARCH', default='resnet50', @@ -102,10 +102,10 @@ help='Where to log') parser.add_argument("--save-frequency", default=5, help="Frequency of checkpoint saving in epochs") -parser.add_argument("--pred-type", default="linear", help="type of the prediction head", +parser.add_argument("--pred-type", default="original", help="type of the prediction head", choices=["linear", "random_linear", "predefined_linear", "low_rank_linear", "original"]) -parser.add_argument("--loss-type", default="cossim", help="type of loss", +parser.add_argument("--loss-type", default="infonce", help="type of loss", choices=["cossim", "infonce", "infonce_scalegrad", "cossim_scalegrad"]) parser.add_argument('--fix-p-size', default=-1.0, type=float, @@ -346,7 +346,9 @@ def cosine_similarioty(x1, x2): def create_loss(type="cossim", fix_p_size=-1): # cossim = CosineSimilarity(inverse=True) - fix_p_term = lambda x: ((torch.norm(x, p=2, dim=-1) - fix_p_size) ** 2).mean() if fix_p_size > 0 else lambda x: 0.0 + fix_p_term = (lambda x: ((torch.norm(x, p=2, dim=-1) - fix_p_size) ** 2).mean()) if fix_p_size > 0 \ + else lambda x: torch.Tensor([0.0]).to(device=x.device) + if "cossim" in type: cossim = torch.nn.CosineSimilarity(dim=1) def compute_all_loss(p1s, p2s, z1, z2): @@ -370,7 +372,7 @@ def compute_all_loss(p1s, p2s, z1, z2): grad_scaling = True else: grad_scaling = False - info_nce = InfoNCE(grad_scaling=grad_scaling) + info_nce = InfoNCE(grad_scaling=grad_scaling) def compute_all_loss(p1s, p2s, z1, z2): losses = {"optim": dict(), "log": dict()} From a2a52cb7e3cca2b17a50e8024ff7b20924837a08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kov=C3=A1cs=20G=C3=A1bor?= Date: Wed, 13 Apr 2022 15:17:39 +0200 Subject: [PATCH 2/6] changing infonce loss --- info_nce.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/info_nce.py b/info_nce.py index 76e948c..c057752 100644 --- a/info_nce.py +++ b/info_nce.py @@ -53,8 +53,17 @@ def forward(self, z1: torch.Tensor, z2: torch.Tensor) -> torch.Tensor: F.normalize(z, dim=1, p=2) if run_type == "top_k_based_on_cos_sim": cos_sim_m = F.softmax(z @ z.T, dim=1) + branch_mask_zeros = torch.zeros((int(cos_sim_m.shape[0]/2), int(cos_sim_m.shape[1]/2))) + branch_mask_ones = torch.ones((int(cos_sim_m.shape[0]/2), int(cos_sim_m.shape[1]/2))) + branch_mask = torch.cat( + [torch.cat([branch_mask_zeros, branch_mask_ones], dim=1), + torch.cat([branch_mask_ones, branch_mask_zeros], dim=1)], + dim=0 + ).to(cos_sim_m.device) + cos_sim_m = torch.mul(branch_mask, cos_sim_m) + top_k, top_k_indices = torch.topk(cos_sim_m, k=10, dim=1) - top_sim = torch.zeros_like(cos_sim_m).scatter(dim=1, index=top_k_indices, src=top_k) + top_sim = torch.zeros_like(cos_sim_m).scatter(dim=1, index=top_k_indices, src=torch.full_like(top_k, 0.75)) labels = torch.max(labels, top_sim) # This is a bit of cheat. Instead of removing cells from From 7b218758ce59e7d10a8054907697d9c94ba692af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kov=C3=A1cs=20G=C3=A1bor?= Date: Wed, 13 Apr 2022 15:18:20 +0200 Subject: [PATCH 3/6] renaming runname --- main_linearpredhead.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main_linearpredhead.py b/main_linearpredhead.py index ea121de..28e970f 100755 --- a/main_linearpredhead.py +++ b/main_linearpredhead.py @@ -41,7 +41,7 @@ and callable(models.__dict__[name])) parser = argparse.ArgumentParser(description='PyTorch ImageNet Training') -parser.add_argument("--runname", default="top_10_based_on_cos_sim", help="Name of run on tensorboard") +parser.add_argument("--runname", default="top_10_based_on_cos_sim_substituted_with_075", help="Name of run on tensorboard") parser.add_argument('--data', metavar='DIR', help='path to dataset', default='/shared_data/imagenet') parser.add_argument('-a', '--arch', metavar='ARCH', default='resnet50', From 7937868d46d296db122c305b4131b63e1e766e1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kov=C3=A1cs=20G=C3=A1bor?= Date: Wed, 4 May 2022 13:30:16 +0200 Subject: [PATCH 4/6] extend info_nce based loss to scheduled topk cossim --- info_nce.py | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/info_nce.py b/info_nce.py index c057752..d25ae8a 100644 --- a/info_nce.py +++ b/info_nce.py @@ -24,9 +24,9 @@ def __init__(self, tau: float = 0.07, grad_scaling=False): self.tau = tau self.grad_scaling = grad_scaling - def forward(self, z1: torch.Tensor, z2: torch.Tensor) -> torch.Tensor: + def forward(self, z1: torch.Tensor, z2: torch.Tensor, epoch) -> torch.Tensor: - run_type = ["label_smoothing", "top_k_based_on_cos_sim"][1] + run_type = ["label_smoothing", "top_k_based_on_cos_sim", "scheduled_cos_sim", "scheduled_top_k_cos_sim"][3] # Collect from all gpu z1 = AllGather.apply(z1) @@ -52,7 +52,9 @@ def forward(self, z1: torch.Tensor, z2: torch.Tensor) -> torch.Tensor: sim_m = z @ z.T F.normalize(z, dim=1, p=2) if run_type == "top_k_based_on_cos_sim": + # pairwise similarities between embeddings cos_sim_m = F.softmax(z @ z.T, dim=1) + # masking for ruling-out pairs from the same siamese branch branch_mask_zeros = torch.zeros((int(cos_sim_m.shape[0]/2), int(cos_sim_m.shape[1]/2))) branch_mask_ones = torch.ones((int(cos_sim_m.shape[0]/2), int(cos_sim_m.shape[1]/2))) branch_mask = torch.cat( @@ -61,10 +63,55 @@ def forward(self, z1: torch.Tensor, z2: torch.Tensor) -> torch.Tensor: dim=0 ).to(cos_sim_m.device) cos_sim_m = torch.mul(branch_mask, cos_sim_m) - + # selecting top-k similarities top_k, top_k_indices = torch.topk(cos_sim_m, k=10, dim=1) top_sim = torch.zeros_like(cos_sim_m).scatter(dim=1, index=top_k_indices, src=torch.full_like(top_k, 0.75)) labels = torch.max(labels, top_sim) + elif run_type == "scheduled_cos_sim": + # pairwise similarities between embeddings + cos_sim_m = F.softmax(z @ z.T, dim=1) + # masking for ruling-out pairs from the same siamese branch + branch_mask_zeros = torch.zeros((int(cos_sim_m.shape[0]/2), int(cos_sim_m.shape[1]/2))) + branch_mask_ones = torch.ones((int(cos_sim_m.shape[0]/2), int(cos_sim_m.shape[1]/2))) + branch_mask = torch.cat( + [torch.cat([branch_mask_zeros, branch_mask_ones], dim=1), + torch.cat([branch_mask_ones, branch_mask_zeros], dim=1)], + dim=0 + ).to(cos_sim_m.device) + cos_sim_m = torch.mul(branch_mask, cos_sim_m) + # scheduling + schedule_start_epoch = 30 + schedule_length_epoch = 20 + if epoch >= schedule_start_epoch: + schedule = torch.linspace(start=0, end=1, steps=schedule_length_epoch) + index_based_on_epoch = min(epoch - schedule_start_epoch, schedule_length_epoch - 1) + labels = (1 - schedule[index_based_on_epoch]) * labels + schedule[index_based_on_epoch] * cos_sim_m + labels = F.softmax(labels, dim=1) + elif run_type == "scheduled_top_k_cos_sim": + # pairwise similarities between embeddings + cos_sim_m = F.softmax(z @ z.T, dim=1) + # masking for ruling-out pairs from the same siamese branch + branch_mask_zeros = torch.zeros((int(cos_sim_m.shape[0] / 2), int(cos_sim_m.shape[1] / 2))) + branch_mask_ones = torch.ones((int(cos_sim_m.shape[0] / 2), int(cos_sim_m.shape[1] / 2))) + branch_mask = torch.cat( + [torch.cat([branch_mask_zeros, branch_mask_ones], dim=1), + torch.cat([branch_mask_ones, branch_mask_zeros], dim=1)], + dim=0 + ).to(cos_sim_m.device) + cos_sim_m = torch.mul(branch_mask, cos_sim_m) + # selecting top-k similarities + top_k, top_k_indices = torch.topk(cos_sim_m, k=10, dim=1) + top_sim = torch.zeros_like(cos_sim_m).scatter(dim=1, index=top_k_indices, src=torch.full_like(top_k, 0.75)) + labels_based_on_similarity = torch.max(labels, top_sim) + # scheduling + schedule_start_epoch = 30 + schedule_length_epoch = 20 + if epoch >= schedule_start_epoch: + schedule = torch.linspace(start=0, end=1, steps=schedule_length_epoch) + index_based_on_epoch = min(epoch - schedule_start_epoch, schedule_length_epoch - 1) + labels = (1 - schedule[index_based_on_epoch]) * labels + schedule[index_based_on_epoch] * labels_based_on_similarity + labels = F.softmax(labels, dim=1) + # This is a bit of cheat. Instead of removing cells from # the matrix where i==j, instead we set it to a very small value From 4ac733c927e977a22f1e4386833d5dc63290aa4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kov=C3=A1cs=20G=C3=A1bor?= Date: Wed, 4 May 2022 13:30:56 +0200 Subject: [PATCH 5/6] adding epoch to loss --- main_linearpredhead.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/main_linearpredhead.py b/main_linearpredhead.py index 28e970f..542a1a0 100755 --- a/main_linearpredhead.py +++ b/main_linearpredhead.py @@ -41,7 +41,7 @@ and callable(models.__dict__[name])) parser = argparse.ArgumentParser(description='PyTorch ImageNet Training') -parser.add_argument("--runname", default="top_10_based_on_cos_sim_substituted_with_075", help="Name of run on tensorboard") +parser.add_argument("--runname", default="schedule_top_10_cos_sim_from30_through20", help="Name of run on tensorboard") parser.add_argument('--data', metavar='DIR', help='path to dataset', default='/shared_data/imagenet') parser.add_argument('-a', '--arch', metavar='ARCH', default='resnet50', @@ -373,11 +373,11 @@ def compute_all_loss(p1s, p2s, z1, z2): else: grad_scaling = False info_nce = InfoNCE(grad_scaling=grad_scaling) - def compute_all_loss(p1s, p2s, z1, z2): + def compute_all_loss(p1s, p2s, z1, z2, epoch): losses = {"optim": dict(), "log": dict()} for i, ((bn_dim, p1), p2) in enumerate(zip(p1s.items(), p2s.values())): - avg_info_nce = (info_nce(p1, z2).mean() + info_nce(p2, z1).mean()) * 0.5 + avg_info_nce = (info_nce(p1, z2, epoch).mean() + info_nce(p2, z1, epoch).mean()) * 0.5 avg_fix_p_term = (fix_p_term(p1) + fix_p_term(p2)) * 0.5 losses["optim"][bn_dim] = avg_info_nce + avg_fix_p_term losses["log"][f"fix_p_term_{bn_dim}"] = avg_fix_p_term @@ -413,7 +413,7 @@ def train(train_loader, model, predictor, criterion, optimizer, epoch, tb_logger # compute output and loss p1, p2, z1, z2 = model(x1=images[0], x2=images[1]) - loss_dict = criterion(p1, p2, z1, z2) + loss_dict = criterion(p1, p2, z1, z2, epoch=epoch) loss = .0 for bn_dim, value in loss_dict["optim"].items(): From 3ab18d95c50a2b92a0b7e4abb280d2e97ba32d1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kov=C3=A1cs=20G=C3=A1bor?= Date: Tue, 31 May 2022 10:30:22 +0200 Subject: [PATCH 6/6] removing true positive from nominator --- info_nce.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/info_nce.py b/info_nce.py index d25ae8a..c3c40bb 100644 --- a/info_nce.py +++ b/info_nce.py @@ -4,6 +4,7 @@ from scale_grad import ScaleGrad from mp_utils import AllGather +import copy __all__ = ['simclr_loss'] @@ -88,6 +89,10 @@ def forward(self, z1: torch.Tensor, z2: torch.Tensor, epoch) -> torch.Tensor: labels = (1 - schedule[index_based_on_epoch]) * labels + schedule[index_based_on_epoch] * cos_sim_m labels = F.softmax(labels, dim=1) elif run_type == "scheduled_top_k_cos_sim": + # hyper-parameters + num_of_top_k = 10 # 10 + schedule_start_epoch = 30 # 30 + schedule_length_epoch = 20 # pairwise similarities between embeddings cos_sim_m = F.softmax(z @ z.T, dim=1) # masking for ruling-out pairs from the same siamese branch @@ -100,25 +105,28 @@ def forward(self, z1: torch.Tensor, z2: torch.Tensor, epoch) -> torch.Tensor: ).to(cos_sim_m.device) cos_sim_m = torch.mul(branch_mask, cos_sim_m) # selecting top-k similarities - top_k, top_k_indices = torch.topk(cos_sim_m, k=10, dim=1) + top_k, top_k_indices = torch.topk(cos_sim_m, k=num_of_top_k, dim=1) top_sim = torch.zeros_like(cos_sim_m).scatter(dim=1, index=top_k_indices, src=torch.full_like(top_k, 0.75)) labels_based_on_similarity = torch.max(labels, top_sim) # scheduling - schedule_start_epoch = 30 - schedule_length_epoch = 20 if epoch >= schedule_start_epoch: schedule = torch.linspace(start=0, end=1, steps=schedule_length_epoch) index_based_on_epoch = min(epoch - schedule_start_epoch, schedule_length_epoch - 1) labels = (1 - schedule[index_based_on_epoch]) * labels + schedule[index_based_on_epoch] * labels_based_on_similarity - labels = F.softmax(labels, dim=1) - + labels = F.normalize(labels, dim=1) # This is a bit of cheat. Instead of removing cells from # the matrix where i==j, instead we set it to a very small value sim_m = sim_m.fill_diagonal_(-10) / self.tau # Get probability distribution - sim_m = torch.nn.functional.log_softmax(sim_m, dim=1) + sim_true_positive = sim_m.clone() + sim_auxiliary_positive = sim_m.clone() + true_positive_mask = torch.roll(torch.eye(sim_m.shape[0]), int(sim_m.shape[0]/2), dims=0).bool().cuda() + sim_auxiliary_positive.masked_fill_(true_positive_mask, (-10 / self.tau)) + sim_true_positive = torch.nn.functional.log_softmax(sim_true_positive, dim=1) + sim_auxiliary_positive = torch.nn.functional.log_softmax(sim_auxiliary_positive, dim=1) + sim_m = torch.where(true_positive_mask.cuda(), sim_true_positive, sim_auxiliary_positive) # Choose values on which we calculate the loss loss = -torch.sum(sim_m * labels) / n