From 037c41bf54977e578d007669792adef52677b41c Mon Sep 17 00:00:00 2001 From: ihahanov Date: Mon, 18 Oct 2021 11:44:48 +0300 Subject: [PATCH 01/67] fix crash when encounter disconnected triangles during training. fix crash when running out of gpu memory add visualization using trimesh --- data/segmentation_data.py | 21 ++++ models/layers/mesh_pool.py | 10 +- models/layers/mesh_prepare.py | 31 ++++-- test.py | 32 ++++++ train.py | 11 +- util/mesh_viewer.py | 5 +- vertex_label_to_edge.py | 204 ++++++++++++++++++++++++++++++++++ 7 files changed, 300 insertions(+), 14 deletions(-) create mode 100644 vertex_label_to_edge.py diff --git a/data/segmentation_data.py b/data/segmentation_data.py index 7d687ae0..e4f5a4f7 100644 --- a/data/segmentation_data.py +++ b/data/segmentation_data.py @@ -5,6 +5,27 @@ import numpy as np from models.layers.mesh import Mesh + +import trimesh as tm +def edges_to_path(edges, color=tm.visual.color.random_color()): + lines = np.asarray(edges) + args = tm.path.exchange.misc.lines_to_path(lines) + colors = [color for _ in range(len(args['entities']))] + path = tm.path.Path3D(**args, colors=colors) + return path + + +def show_mesh(mesh, label, colors=[[0,0,0,255], [120,120,120,255]]): + colors = np.array(colors) + edges = mesh.vs[mesh.edges] + tm.Scene([edges_to_path(e, colors[int(l)]) for e, l in zip(edges, label)]).show() + + +def show_vertices(mesh, label, colors=[[0,0,0,255], [120,120,120,255]]): + colors = np.array(colors) + tm.PointCloud(mesh.vs, colors=np.array(colors)[label]).show() + + class SegmentationData(BaseDataset): def __init__(self, opt): diff --git a/models/layers/mesh_pool.py b/models/layers/mesh_pool.py index 394d0fc9..b903ceab 100644 --- a/models/layers/mesh_pool.py +++ b/models/layers/mesh_pool.py @@ -56,6 +56,7 @@ def __pool_main(self, mesh_index): self.__updated_fe[mesh_index] = fe def __pool_edge(self, mesh, edge_id, mask, edge_groups): + # if the edge is a boundary edge or any of its neighbor edges are boundary if self.has_boundaries(mesh, edge_id): return False elif self.__clean_side(mesh, edge_id, mask, edge_groups, 0)\ @@ -74,7 +75,7 @@ def __pool_edge(self, mesh, edge_id, mask, edge_groups): def __clean_side(self, mesh, edge_id, mask, edge_groups, side): if mesh.edges_count <= self.__out_target: return False - invalid_edges = MeshPool.__get_invalids(mesh, edge_id, edge_groups, side) + invalid_edges = MeshPool.__get_invalids(mesh, edge_id, edge_groups, side) # triplet edges sharing the same vertex while len(invalid_edges) != 0 and mesh.edges_count > self.__out_target: self.__remove_triplete(mesh, mask, edge_groups, invalid_edges) if mesh.edges_count <= self.__out_target: @@ -116,6 +117,12 @@ def __pool_side(self, mesh, edge_id, mask, edge_groups, side): def __get_invalids(mesh, edge_id, edge_groups, side): info = MeshPool.__get_face_info(mesh, edge_id, side) key_a, key_b, side_a, side_b, other_side_a, other_side_b, other_keys_a, other_keys_b = info + + # if we have a separate triangle not connected to anything + if len(set(other_keys_a).intersection([key_a, key_b, edge_id])) == 2 or \ + len(set(other_keys_b).intersection([key_a, key_b, edge_id])) == 2: + return [] + shared_items = MeshPool.__get_shared_items(other_keys_a, other_keys_b) if len(shared_items) == 0: return [] @@ -135,6 +142,7 @@ def __get_invalids(mesh, edge_id, edge_groups, side): MeshPool.__union_groups(mesh, edge_groups, middle_edge, update_key_a) MeshPool.__union_groups(mesh, edge_groups, key_b, update_key_b) MeshPool.__union_groups(mesh, edge_groups, middle_edge, update_key_b) + return [key_a, key_b, middle_edge] @staticmethod diff --git a/models/layers/mesh_prepare.py b/models/layers/mesh_prepare.py index 47e827c7..aba184e0 100644 --- a/models/layers/mesh_prepare.py +++ b/models/layers/mesh_prepare.py @@ -11,11 +11,12 @@ def fill_mesh(mesh2fill, file: str, opt): mesh_data = from_scratch(file, opt) np.savez_compressed(load_path, gemm_edges=mesh_data.gemm_edges, vs=mesh_data.vs, edges=mesh_data.edges, edges_count=mesh_data.edges_count, ve=mesh_data.ve, v_mask=mesh_data.v_mask, - filename=mesh_data.filename, sides=mesh_data.sides, + filename=mesh_data.filename, sides=mesh_data.sides, faces=mesh_data.faces, edge_lengths=mesh_data.edge_lengths, edge_areas=mesh_data.edge_areas, features=mesh_data.features) mesh2fill.vs = mesh_data['vs'] mesh2fill.edges = mesh_data['edges'] + mesh2fill.faces = mesh_data['faces'] mesh2fill.gemm_edges = mesh_data['gemm_edges'] mesh2fill.edges_count = int(mesh_data['edges_count']) mesh2fill.ve = mesh_data['ve'] @@ -51,9 +52,9 @@ def __getitem__(self, item): mesh_data.filename = 'unknown' mesh_data.edge_lengths = None mesh_data.edge_areas = [] - mesh_data.vs, faces = fill_from_file(mesh_data, file) + mesh_data.vs, mesh_data.faces = fill_from_file(mesh_data, file) mesh_data.v_mask = np.ones(len(mesh_data.vs), dtype=bool) - faces, face_areas = remove_non_manifolds(mesh_data, faces) + faces, face_areas = remove_non_manifolds(mesh_data, mesh_data.faces) if opt.num_aug > 1: faces = augmentation(mesh_data, opt, faces) build_gemm(mesh_data, faces, face_areas) @@ -155,7 +156,7 @@ def build_gemm(mesh, faces, face_areas): sides[edge_key][nb_count[edge_key] - 2] = nb_count[edge2key[faces_edges[(idx + 1) % 3]]] - 1 sides[edge_key][nb_count[edge_key] - 1] = nb_count[edge2key[faces_edges[(idx + 2) % 3]]] - 2 mesh.edges = np.array(edges, dtype=np.int32) - mesh.gemm_edges = np.array(edge_nb, dtype=np.int64) + mesh.gemm_edges = np.array(edge_nb, dtype=np.int64) # [n_edges, 4] - matrix of edges and 4 their neighbors mesh.sides = np.array(sides, dtype=np.int64) mesh.edges_count = edges_count mesh.edge_areas = np.array(mesh.edge_areas, dtype=np.float32) / np.sum(face_areas) #todo whats the difference between edge_areas and edge_lenghts? @@ -195,10 +196,12 @@ def slide_verts(mesh, prct): for vi in vids: if shifted < target: edges = mesh.ve[vi] - if min(dihedral[edges]) > 2.65: - edge = mesh.edges[np.random.choice(edges)] - vi_t = edge[1] if vi == edge[0] else edge[0] - nv = mesh.vs[vi] + np.random.uniform(0.2, 0.5) * (mesh.vs[vi_t] - mesh.vs[vi]) + if len(dihedral[edges]) == 0: + continue + if min(dihedral[edges]) > 2.65: # if any 2 adjacent faces for the vi vertex are flat enough + edge = mesh.edges[np.random.choice(edges)] # take one random edge + vi_t = edge[1] if vi == edge[0] else edge[0] # take the opposite vertex + nv = mesh.vs[vi] + np.random.uniform(0.2, 0.5) * (mesh.vs[vi_t] - mesh.vs[vi]) # shift origin vi vertex mesh.vs[vi] = nv shifted += 1 else: @@ -366,18 +369,26 @@ def get_edge_points(mesh): def get_side_points(mesh, edge_id): + """ + Return 4 points indices for each edge. 2 point indices constituting the edge itself, and 2 point indices on the + opposite sides of both triangles sharing the given edge. + [edge_a[0], edge_a[1], opposite_vertex_A, opposite_vertex_B] + + In case the edge lies on the boundary, the third and fourth edge are similar and correspond to the same vertex. + [edge_a[0], edge_a[1], opposite_vertex_A, opposite_vertex_A] + """ # if mesh.gemm_edges[edge_id, side] == -1: # return mesh.get_side_points(edge_id, ((side + 2) % 4)) # else: edge_a = mesh.edges[edge_id] - if mesh.gemm_edges[edge_id, 0] == -1: + if mesh.gemm_edges[edge_id, 0] == -1: # If edge lies on the boundary with LEFT face missing edge_b = mesh.edges[mesh.gemm_edges[edge_id, 2]] edge_c = mesh.edges[mesh.gemm_edges[edge_id, 3]] else: edge_b = mesh.edges[mesh.gemm_edges[edge_id, 0]] edge_c = mesh.edges[mesh.gemm_edges[edge_id, 1]] - if mesh.gemm_edges[edge_id, 2] == -1: + if mesh.gemm_edges[edge_id, 2] == -1: # If edge lies on the boundary with RIGHT face missing edge_d = mesh.edges[mesh.gemm_edges[edge_id, 0]] edge_e = mesh.edges[mesh.gemm_edges[edge_id, 1]] else: diff --git a/test.py b/test.py index 15492f5b..a8ae9652 100644 --- a/test.py +++ b/test.py @@ -2,6 +2,26 @@ from data import DataLoader from models import create_model from util.writer import Writer +import trimesh as tm +import numpy as np +from copy import deepcopy + + +import trimesh as tm +def edges_to_path(edges, color=tm.visual.color.random_color()): + lines = np.asarray(edges) + args = tm.path.exchange.misc.lines_to_path(lines) + colors = [color for _ in range(len(args['entities']))] + path = tm.path.Path3D(**args, colors=colors) + return path + + +def show_edges(mesh, label, colors=[[0,0,0,255], [120,120,120,255]]): + colors = np.array(colors) + edges = mesh.vs[mesh.edges] + tm.Scene([edges_to_path(e, colors[int(l)]) for e, l in zip(edges, label)]).show() + + def run_test(epoch=-1): @@ -14,7 +34,19 @@ def run_test(epoch=-1): # test writer.reset_counter() for i, data in enumerate(dataset): + mesh = deepcopy(data['mesh'][0]) model.set_input(data) + pred_class = model.forward().max(1)[1] + # show_mesh(mesh, pred_class[0]) + edges = mesh.edges + vertices = mesh.vs + vertex_label = np.zeros(len(vertices)) + for e_l, e in zip(pred_class[0], edges): + if e_l == 1: + vertex_label[e] = 1 + faces = mesh.faces + vertex_colors = np.array([tm.visual.random_color(), tm.visual.random_color()])[vertex_label.astype(int)] + tm.Trimesh(faces=faces, vertices=vertices, vertex_colors=vertex_colors).show() ncorrect, nexamples = model.test() writer.update_counter(ncorrect, nexamples) writer.print_acc(epoch, writer.acc) diff --git a/train.py b/train.py index 41b326b7..4ea16d60 100644 --- a/train.py +++ b/train.py @@ -4,6 +4,8 @@ from models import create_model from util.writer import Writer from test import run_test +import torch + if __name__ == '__main__': opt = TrainOptions().parse() @@ -21,6 +23,11 @@ epoch_iter = 0 for i, data in enumerate(dataset): + + # for mesh, label in zip(data['mesh'], data['label']): + # mesh.export_folder = '/home/ihahanov/Projects/meshcnn/datasets/roof_seg/test' + # mesh.export_segments(label) + iter_start_time = time.time() if total_steps % opt.print_freq == 0: t_data = iter_start_time - iter_data_time @@ -45,7 +52,7 @@ print('saving the model at the end of epoch %d, iters %d' % (epoch, total_steps)) model.save_network('latest') - model.save_network(epoch) + # model.save_network(epoch) print('End of epoch %d / %d \t Time Taken: %d sec' % (epoch, opt.niter + opt.niter_decay, time.time() - epoch_start_time)) @@ -57,4 +64,6 @@ acc = run_test(epoch) writer.plot_acc(acc, epoch) + torch.cuda.empty_cache() + writer.close() diff --git a/util/mesh_viewer.py b/util/mesh_viewer.py index c7214fb8..b0d34dc5 100644 --- a/util/mesh_viewer.py +++ b/util/mesh_viewer.py @@ -118,6 +118,8 @@ def fix_vertices(): if len(splitted_line) >= 4: edge_v = [int(c) - 1 for c in splitted_line[1:-1]] edge_c = int(splitted_line[-1]) + if edge_c < 0: + continue add_to_edges() vs = V(vs) @@ -142,8 +144,7 @@ def view_meshes(*files, offset=.2): if __name__=='__main__': import argparse parser = argparse.ArgumentParser("view meshes") - parser.add_argument('--files', nargs='+', default=['checkpoints/human_seg/meshes/shrec__14_0.obj', - 'checkpoints/human_seg/meshes/shrec__14_3.obj'], type=str, + parser.add_argument('--files', nargs='+', default=['/home/ihahanov/Projects/meshcnn/checkpoints/roof_seg/meshes/basnett_0.obj'], type=str, help="list of 1 or more .obj files") args = parser.parse_args() diff --git a/vertex_label_to_edge.py b/vertex_label_to_edge.py new file mode 100644 index 00000000..80413124 --- /dev/null +++ b/vertex_label_to_edge.py @@ -0,0 +1,204 @@ +import numpy as np +import os +import glob +import filecmp +import sys + + +''' +Creates esseg files for accuracy with smooth transitions between classes +Requires Objects and corresponding labels per edge +Author: Rana Hanocka / Lisa Schneider + +@input: + path where seg, sseg, train, test folders are placed + +@output: + esseg files for all objects + to run it from cmd line: + python create_sseg.py /home/user/MedMeshCNN/datasets/human_seg/ +''' + +def compute_face_normals_and_areas(vs, faces): + face_normals = np.cross(vs[faces[:, 1]] - vs[faces[:, 0]], + vs[faces[:, 2]] - vs[faces[:, 1]]) + face_areas = np.sqrt((face_normals ** 2).sum(axis=1)) + face_normals /= face_areas[:, np.newaxis] + assert (not np.any(face_areas[:, np.newaxis] == 0)), 'has zero area face' + face_areas *= 0.5 + return face_normals, face_areas + + +def remove_non_manifolds(vs, faces): + edges_set = set() + mask = np.ones(len(faces), dtype=bool) + _, face_areas = compute_face_normals_and_areas(vs, faces) + for face_id, face in enumerate(faces): + if face_areas[face_id] == 0: + mask[face_id] = False + continue + faces_edges = [] + is_manifold = False + for i in range(3): + cur_edge = (face[i], face[(i + 1) % 3]) + if cur_edge in edges_set: + is_manifold = True + break + else: + faces_edges.append(cur_edge) + if is_manifold: + mask[face_id] = False + else: + for idx, edge in enumerate(faces_edges): + edges_set.add(edge) + return faces[mask], face_areas[mask] + +def get_gemm_edges(faces, export_name_edges): + """ + gemm_edges: array (#E x 4) of the 4 one-ring neighbors for each edge + sides: array (#E x 4) indices (values of: 0,1,2,3) indicating where an edge is in the gemm_edge entry of the 4 neighboring edges + for example edge i -> gemm_edges[gemm_edges[i], sides[i]] == [i, i, i, i] + """ + edge_nb = [] + sides = [] + edge2key = dict() + edges = [] + edges_count = 0 + nb_count = [] + for face_id, face in enumerate(faces): + faces_edges = [] + for i in range(3): + cur_edge = (face[i], face[(i + 1) % 3]) + faces_edges.append(cur_edge) + for idx, edge in enumerate(faces_edges): + edge = tuple(sorted(list(edge))) + faces_edges[idx] = edge + if edge not in edge2key: + edge2key[edge] = edges_count + edges.append(list(edge)) + edge_nb.append([-1, -1, -1, -1]) + sides.append([-1, -1, -1, -1]) + nb_count.append(0) + edges_count += 1 + for idx, edge in enumerate(faces_edges): + edge_key = edge2key[edge] + edge_nb[edge_key][nb_count[edge_key]] = edge2key[faces_edges[(idx + 1) % 3]] + edge_nb[edge_key][nb_count[edge_key] + 1] = edge2key[faces_edges[(idx + 2) % 3]] + nb_count[edge_key] += 2 + for idx, edge in enumerate(faces_edges): + edge_key = edge2key[edge] + sides[edge_key][nb_count[edge_key] - 2] = nb_count[edge2key[faces_edges[(idx + 1) % 3]]] - 1 + sides[edge_key][nb_count[edge_key] - 1] = nb_count[edge2key[faces_edges[(idx + 2) % 3]]] - 2 + edges = np.array(edges, dtype=np.int32) + np.savetxt(export_name_edges, edges, fmt='%i') + return edge_nb, edges + + +def load_faces(path): + with open(path, 'r') as f: + for line in f: + inner_list = [vertices.strip() for vertices in line.split(',')] + gemm_edges.append(inner_list) + return gemm_edges + + +def load_labels(path): + with open(path, 'r') as f: + content = f.read().splitlines() + return content + +def create_sseg_file(gemms, labels, export_name_seseg): + gemmlabels = {} + classes = len(np.unique(labels)) + class_to_idx = {v: i for i, v in enumerate(np.unique(labels))} + totaledges = len(gemms) + sseg = np.zeros([ totaledges, classes]) + for i, edges in enumerate(gemms): + alllabels = [] + for edge in range(len(edges)): + lookupEdge = edges[edge] + label = labels[lookupEdge] + alllabels.append(label) + gemmlabels[i] = alllabels + + for i, edges in enumerate(gemms): + gemmlab = gemmlabels[i] + uniqueValues, counts = np.unique(gemmlab, return_counts=True) + for j, label in enumerate(uniqueValues): + weight = 0.125*counts[j] + sseg[i][class_to_idx[label]] = weight + np.savetxt(export_name_seseg, sseg, fmt='%1.6f') + +def get_obj(file): + vs, faces = [], [] + f = open(file) + for line in f: + line = line.strip() + splitted_line = line.split() + if not splitted_line: + continue + elif splitted_line[0] == 'v': + vs.append([float(v) for v in splitted_line[1:4]]) + elif splitted_line[0] == 'f': + face_vertex_ids = [int(c.split('/')[0]) for c in splitted_line[1:]] + assert len(face_vertex_ids) == 3 + face_vertex_ids = [(ind - 1) if (ind >= 0) else (len(vs) + ind) + for ind in face_vertex_ids] + faces.append(face_vertex_ids) + f.close() + vs = np.asarray(vs) + faces = np.asarray(faces, dtype=int) + assert np.logical_and(faces >= 0, faces < len(vs)).all() + return faces, vs + + +import trimesh as tm +def edges_to_path(edges, color=tm.visual.color.random_color()): + lines = np.asarray(edges) + args = tm.path.exchange.misc.lines_to_path(lines) + colors = [color for _ in range(len(args['entities']))] + path = tm.path.Path3D(**args, colors=colors) + return path + + +def show_mesh(edges, vs, label, colors=[[0,0,0,255], [120,120,120,255]]): + colors = np.array(colors) + edges = vs[edges] + tm.Scene([edges_to_path(e, colors[int(l)]) for e, l in zip(edges, label)]).show() + + + +def create_files(path): + print("path", glob.glob(os.path.join(path, 'train/*.obj'))) + for filename in glob.glob(os.path.join(path, 'train/*.obj')): + basename = os.path.splitext(os.path.basename(filename))[0] + v_label_name = os.path.join(os.path.join(path, 'seg'), basename + '_v.eseg') + label_name = os.path.join(os.path.join(path, 'seg'), basename + '.eseg') + export_name_seseg = os.path.join(os.path.join(path, 'sseg'), basename + '.seseg') + export_name_edges = os.path.join(os.path.join(path, 'edges'), basename + '.edges') + + faces, vs = get_obj(filename) + faces, face_areas = remove_non_manifolds(vs, faces) + gemms, edges = get_gemm_edges(faces, export_name_edges) + with open(v_label_name) as f: + v_label = np.array(f.readlines(), dtype=int) + + edge_label = [] + for e in edges: + if v_label[e[0]] == 1 and v_label[e[1]] == 1: + edge_label.append(str(2)) + else: + edge_label.append(str(1)) + + with open(label_name, 'w') as f: + f.write('\n'.join(edge_label)) + + if os.path.isfile(label_name): + + create_sseg_file(gemms, edge_label, export_name_seseg) + else: + print(label_name, "is no directory") + + +if __name__ == '__main__': + create_files(sys.argv[1]) \ No newline at end of file From e4695ad7cd804ddff8ae760fb26dde13b0096f72 Mon Sep 17 00:00:00 2001 From: ihahanov Date: Mon, 25 Oct 2021 16:19:55 +0300 Subject: [PATCH 02/67] add scripts for preparing blender data for traininig --- data/blender_scripts/extract_vertex_labels.py | 15 ++ data/make_annotation_from_vertex_labels.py | 196 ++++++++++++++++++ test.py | 27 +-- train.py | 4 - 4 files changed, 225 insertions(+), 17 deletions(-) create mode 100644 data/blender_scripts/extract_vertex_labels.py create mode 100644 data/make_annotation_from_vertex_labels.py diff --git a/data/blender_scripts/extract_vertex_labels.py b/data/blender_scripts/extract_vertex_labels.py new file mode 100644 index 00000000..36eef2c5 --- /dev/null +++ b/data/blender_scripts/extract_vertex_labels.py @@ -0,0 +1,15 @@ + +import bpy + +ob = bpy.context.object +obdata = bpy.context.object.data + +label = [] +for v in obdata.vertices: + if bpy.context.object.vertex_groups['roof'].index in [i.group for i in v.groups]: + label.append(str(1)) + else: + label.append(str(0)) + +with open('/home/ihahanov/Projects/meshcnn/datasets/roof_seg/vseg/2620 Benefield Simplified + Annotated.eseg', 'w') as f: + f.write('\n'.join(label)) \ No newline at end of file diff --git a/data/make_annotation_from_vertex_labels.py b/data/make_annotation_from_vertex_labels.py new file mode 100644 index 00000000..9f1bcf32 --- /dev/null +++ b/data/make_annotation_from_vertex_labels.py @@ -0,0 +1,196 @@ +import numpy as np +import os +import glob +import filecmp +import sys + + +''' +Creates esseg files for accuracy with smooth transitions between classes +Requires Objects and corresponding labels per edge +Author: Rana Hanocka / Lisa Schneider + +@input: + path where seg, sseg, train, test folders are placed + +@output: + esseg files for all objects + to run it from cmd line: + python create_sseg.py /home/user/MedMeshCNN/datasets/human_seg/ +''' + +def compute_face_normals_and_areas(vs, faces): + face_normals = np.cross(vs[faces[:, 1]] - vs[faces[:, 0]], + vs[faces[:, 2]] - vs[faces[:, 1]]) + face_areas = np.sqrt((face_normals ** 2).sum(axis=1)) + face_normals /= face_areas[:, np.newaxis] + assert (not np.any(face_areas[:, np.newaxis] == 0)), 'has zero area face' + face_areas *= 0.5 + return face_normals, face_areas + + +def remove_non_manifolds(vs, faces): + edges_set = set() + mask = np.ones(len(faces), dtype=bool) + _, face_areas = compute_face_normals_and_areas(vs, faces) + for face_id, face in enumerate(faces): + if face_areas[face_id] == 0: + mask[face_id] = False + continue + faces_edges = [] + is_manifold = False + for i in range(3): + cur_edge = (face[i], face[(i + 1) % 3]) + if cur_edge in edges_set: + is_manifold = True + break + else: + faces_edges.append(cur_edge) + if is_manifold: + mask[face_id] = False + else: + for idx, edge in enumerate(faces_edges): + edges_set.add(edge) + return faces[mask], face_areas[mask] + +def get_gemm_edges(faces, export_name_edges): + """ + gemm_edges: array (#E x 4) of the 4 one-ring neighbors for each edge + sides: array (#E x 4) indices (values of: 0,1,2,3) indicating where an edge is in the gemm_edge entry of the 4 neighboring edges + for example edge i -> gemm_edges[gemm_edges[i], sides[i]] == [i, i, i, i] + """ + edge_nb = [] + sides = [] + edge2key = dict() + edges = [] + edges_count = 0 + nb_count = [] + for face_id, face in enumerate(faces): + faces_edges = [] + for i in range(3): + cur_edge = (face[i], face[(i + 1) % 3]) + faces_edges.append(cur_edge) + for idx, edge in enumerate(faces_edges): + edge = tuple(sorted(list(edge))) + faces_edges[idx] = edge + if edge not in edge2key: + edge2key[edge] = edges_count + edges.append(list(edge)) + edge_nb.append([-1, -1, -1, -1]) + sides.append([-1, -1, -1, -1]) + nb_count.append(0) + edges_count += 1 + for idx, edge in enumerate(faces_edges): + edge_key = edge2key[edge] + edge_nb[edge_key][nb_count[edge_key]] = edge2key[faces_edges[(idx + 1) % 3]] + edge_nb[edge_key][nb_count[edge_key] + 1] = edge2key[faces_edges[(idx + 2) % 3]] + nb_count[edge_key] += 2 + for idx, edge in enumerate(faces_edges): + edge_key = edge2key[edge] + sides[edge_key][nb_count[edge_key] - 2] = nb_count[edge2key[faces_edges[(idx + 1) % 3]]] - 1 + sides[edge_key][nb_count[edge_key] - 1] = nb_count[edge2key[faces_edges[(idx + 2) % 3]]] - 2 + edges = np.array(edges, dtype=np.int32) + np.savetxt(export_name_edges, edges, fmt='%i') + return edge_nb, edges + + +def load_labels(path): + with open(path, 'r') as f: + content = f.read().splitlines() + return content + +def create_sseg_file(gemms, labels, export_name_seseg): + gemmlabels = {} + classes = len(np.unique(labels)) + class_to_idx = {v: i for i, v in enumerate(np.unique(labels))} + totaledges = len(gemms) + sseg = np.zeros([ totaledges, classes]) + for i, edges in enumerate(gemms): + alllabels = [] + for edge in range(len(edges)): + lookupEdge = edges[edge] + label = labels[lookupEdge] + alllabels.append(label) + gemmlabels[i] = alllabels + + for i, edges in enumerate(gemms): + gemmlab = gemmlabels[i] + uniqueValues, counts = np.unique(gemmlab, return_counts=True) + for j, label in enumerate(uniqueValues): + weight = 0.125*counts[j] + sseg[i][class_to_idx[label]] = weight + np.savetxt(export_name_seseg, sseg, fmt='%1.6f') + +def get_obj(file): + vs, faces = [], [] + f = open(file) + for line in f: + line = line.strip() + splitted_line = line.split() + if not splitted_line: + continue + elif splitted_line[0] == 'v': + vs.append([float(v) for v in splitted_line[1:4]]) + elif splitted_line[0] == 'f': + face_vertex_ids = [int(c.split('/')[0]) for c in splitted_line[1:]] + assert len(face_vertex_ids) == 3 + face_vertex_ids = [(ind - 1) if (ind >= 0) else (len(vs) + ind) + for ind in face_vertex_ids] + faces.append(face_vertex_ids) + f.close() + vs = np.asarray(vs) + faces = np.asarray(faces, dtype=int) + assert np.logical_and(faces >= 0, faces < len(vs)).all() + return faces, vs + + +import trimesh as tm +def edges_to_path(edges, color=tm.visual.color.random_color()): + lines = np.asarray(edges) + args = tm.path.exchange.misc.lines_to_path(lines) + colors = [color for _ in range(len(args['entities']))] + path = tm.path.Path3D(**args, colors=colors) + return path + + +def show_mesh(edges, vs, label, colors=[[0,0,0,255], [120,120,120,255]]): + colors = np.array(colors) + edges = vs[edges] + tm.Scene([edges_to_path(e, colors[int(l)]) for e, l in zip(edges, label)]).show() + + + +def create_files(path): + print("path", glob.glob(os.path.join(path, 'train/*.obj'))) + for filename in glob.glob(os.path.join(path, 'train/*.obj')): + basename = os.path.splitext(os.path.basename(filename))[0] + v_label_name = os.path.join(os.path.join(path, 'vseg'), basename + '.eseg') + label_name = os.path.join(os.path.join(path, 'seg'), basename + '.eseg') + export_name_seseg = os.path.join(os.path.join(path, 'sseg'), basename + '.seseg') + export_name_edges = os.path.join(os.path.join(path, 'edges'), basename + '.edges') + + faces, vs = get_obj(filename) + faces, face_areas = remove_non_manifolds(vs, faces) + gemms, edges = get_gemm_edges(faces, export_name_edges) + with open(v_label_name) as f: + v_label = np.array(f.readlines(), dtype=int) + + edge_label = [] + for e in edges: + if v_label[e[0]] == 1 and v_label[e[1]] == 1: + edge_label.append(str(2)) + else: + edge_label.append(str(1)) + + with open(label_name, 'w') as f: + f.write('\n'.join(edge_label)) + print(len(edge_label)) + if os.path.isfile(label_name): + + create_sseg_file(gemms, edge_label, export_name_seseg) + else: + print(label_name, "is no directory") + + +if __name__ == '__main__': + create_files(sys.argv[1]) \ No newline at end of file diff --git a/test.py b/test.py index a8ae9652..ed6179c8 100644 --- a/test.py +++ b/test.py @@ -22,8 +22,6 @@ def show_edges(mesh, label, colors=[[0,0,0,255], [120,120,120,255]]): tm.Scene([edges_to_path(e, colors[int(l)]) for e, l in zip(edges, label)]).show() - - def run_test(epoch=-1): print('Running Test') opt = TestOptions().parse() @@ -36,18 +34,21 @@ def run_test(epoch=-1): for i, data in enumerate(dataset): mesh = deepcopy(data['mesh'][0]) model.set_input(data) - pred_class = model.forward().max(1)[1] - # show_mesh(mesh, pred_class[0]) - edges = mesh.edges - vertices = mesh.vs - vertex_label = np.zeros(len(vertices)) - for e_l, e in zip(pred_class[0], edges): - if e_l == 1: - vertex_label[e] = 1 - faces = mesh.faces - vertex_colors = np.array([tm.visual.random_color(), tm.visual.random_color()])[vertex_label.astype(int)] - tm.Trimesh(faces=faces, vertices=vertices, vertex_colors=vertex_colors).show() + + # pred_class = model.forward().max(1)[1] + # # show_mesh(mesh, pred_class[0]) + # edges = mesh.edges + # vertices = mesh.vs + # vertex_label = np.zeros(len(vertices)) + # for e_l, e in zip(pred_class[0], edges): + # if e_l == 1: + # vertex_label[e] = 1 + # faces = mesh.faces + # vertex_colors = np.array([tm.visual.random_color(), tm.visual.random_color()])[vertex_label.astype(int)] + # tm.Trimesh(faces=faces, vertices=vertices, vertex_colors=vertex_colors).show() + ncorrect, nexamples = model.test() + writer.update_counter(ncorrect, nexamples) writer.print_acc(epoch, writer.acc) return writer.acc diff --git a/train.py b/train.py index 4ea16d60..ee778656 100644 --- a/train.py +++ b/train.py @@ -24,10 +24,6 @@ for i, data in enumerate(dataset): - # for mesh, label in zip(data['mesh'], data['label']): - # mesh.export_folder = '/home/ihahanov/Projects/meshcnn/datasets/roof_seg/test' - # mesh.export_segments(label) - iter_start_time = time.time() if total_steps % opt.print_freq == 0: t_data = iter_start_time - iter_data_time From 5b53bcb10098b97a5470fdf1c8a865e3f95af7e4 Mon Sep 17 00:00:00 2001 From: ihahanov Date: Thu, 4 Nov 2021 18:59:34 +0200 Subject: [PATCH 03/67] add inference classifier --- options/base_options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/options/base_options.py b/options/base_options.py index 61f21ce0..09b9aa26 100644 --- a/options/base_options.py +++ b/options/base_options.py @@ -11,7 +11,7 @@ def __init__(self): def initialize(self): # data params - self.parser.add_argument('--dataroot', required=True, help='path to meshes (should have subfolders train, test)') + self.parser.add_argument('--dataroot', required=False, help='path to meshes (should have subfolders train, test)') self.parser.add_argument('--dataset_mode', choices={"classification", "segmentation"}, default='classification') self.parser.add_argument('--ninput_edges', type=int, default=750, help='# of input edges (will include dummy edges)') self.parser.add_argument('--max_dataset_size', type=int, default=float("inf"), help='Maximum number of samples per epoch') From 3afd908bcd07b2ddd1ebd481722fdfecda00942c Mon Sep 17 00:00:00 2001 From: ihahanov Date: Mon, 8 Nov 2021 16:25:26 +0200 Subject: [PATCH 04/67] fix predictions in roof_extraction meshcnn --- visualize.py | 54 ++++++++++++++++++++++++++++++++++++++++ visualize_annotations.py | 37 +++++++++++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 visualize.py create mode 100644 visualize_annotations.py diff --git a/visualize.py b/visualize.py new file mode 100644 index 00000000..9a9fdcaf --- /dev/null +++ b/visualize.py @@ -0,0 +1,54 @@ +from options.test_options import TestOptions +from data import DataLoader +from models import create_model +from util.writer import Writer +import trimesh as tm +import numpy as np +from copy import deepcopy +from data.segmentation_data import show_mesh +import trimesh as tm + + +def edges_to_path(edges, color=tm.visual.color.random_color()): + lines = np.asarray(edges) + args = tm.path.exchange.misc.lines_to_path(lines) + colors = [color for _ in range(len(args['entities']))] + path = tm.path.Path3D(**args, colors=colors) + return path + + +def show_edges(mesh, label, colors=[[0, 0, 0, 255], [120, 120, 120, 255]]): + colors = np.array(colors) + edges = mesh.vs[mesh.edges] + tm.Scene([edges_to_path(e, colors[int(l)]) for e, l in zip(edges, label)]).show() + + +def run_test(epoch=-1): + print('Running Test') + opt = TestOptions().parse() + opt.serial_batches = True # no shuffle + dataset = DataLoader(opt) + model = create_model(opt) + writer = Writer(opt) + # test + writer.reset_counter() + for i, data in enumerate(dataset): + mesh = deepcopy(data['mesh'][0]) + + show_mesh(mesh, data['label'][0]) + # model.set_input(data) + # + pred_class = model.forward().max(1)[1] + # show_mesh(mesh, pred_class[0]) + edges = mesh.edges + vertices = mesh.vs + vertex_label = np.zeros(len(vertices)) + for e_l, e in zip(pred_class[0], edges): + if e_l == 1: + vertex_label[e] = 1 + faces = mesh.faces + vertex_colors = np.array([[255,100,0,255], [0,100,255,255]])[vertex_label.astype(int)] + tm.Trimesh(faces=faces, vertices=vertices, vertex_colors=vertex_colors).show() + +if __name__ == '__main__': + run_test() diff --git a/visualize_annotations.py b/visualize_annotations.py new file mode 100644 index 00000000..955dee2d --- /dev/null +++ b/visualize_annotations.py @@ -0,0 +1,37 @@ +from options.test_options import TestOptions +from data import DataLoader +from models import create_model +from util.writer import Writer +import trimesh as tm +import numpy as np +from copy import deepcopy +from data.segmentation_data import show_mesh +import trimesh as tm + + +def edges_to_path(edges, color=tm.visual.color.random_color()): + lines = np.asarray(edges) + args = tm.path.exchange.misc.lines_to_path(lines) + colors = [color for _ in range(len(args['entities']))] + path = tm.path.Path3D(**args, colors=colors) + return path + + +def show_edges(mesh, label, colors=[[0, 0, 0, 255], [120, 120, 120, 255]]): + colors = np.array(colors) + edges = mesh.vs[mesh.edges] + tm.Scene([edges_to_path(e, colors[int(l)]) for e, l in zip(edges, label)]).show() + + +def run_test(epoch=-1): + print('Running Test') + opt = TestOptions().parse() + opt.serial_batches = True # no shuffle + dataset = DataLoader(opt) + for i, data in enumerate(dataset): + mesh = deepcopy(data['mesh'][0]) + + show_mesh(mesh, data['label'][0]) + +if __name__ == '__main__': + run_test() From 981cc6ee1e1b606e36b17ff5c97764ba91060e2b Mon Sep 17 00:00:00 2001 From: ihahanov Date: Wed, 17 Nov 2021 12:47:17 +0200 Subject: [PATCH 05/67] refactor visualization --- visualize.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/visualize.py b/visualize.py index 9a9fdcaf..23558b5e 100644 --- a/visualize.py +++ b/visualize.py @@ -23,6 +23,17 @@ def show_edges(mesh, label, colors=[[0, 0, 0, 255], [120, 120, 120, 255]]): tm.Scene([edges_to_path(e, colors[int(l)]) for e, l in zip(edges, label)]).show() +def show_mesh(mesh, label): + edges = mesh.edges + vertices = mesh.vs + vertex_label = np.zeros(len(vertices)) + for e_l, e in zip(label[0], edges): + if e_l == 1: + vertex_label[e] = 1 + faces = mesh.faces + vertex_colors = np.array([[255, 100, 0, 255], [0, 100, 255, 255]])[vertex_label.astype(int)] + tm.Trimesh(faces=faces, vertices=vertices, vertex_colors=vertex_colors).show() + def run_test(epoch=-1): print('Running Test') opt = TestOptions().parse() @@ -35,20 +46,12 @@ def run_test(epoch=-1): for i, data in enumerate(dataset): mesh = deepcopy(data['mesh'][0]) - show_mesh(mesh, data['label'][0]) - # model.set_input(data) + # show_mesh(mesh, data['label'][0]) + model.set_input(data) # pred_class = model.forward().max(1)[1] # show_mesh(mesh, pred_class[0]) - edges = mesh.edges - vertices = mesh.vs - vertex_label = np.zeros(len(vertices)) - for e_l, e in zip(pred_class[0], edges): - if e_l == 1: - vertex_label[e] = 1 - faces = mesh.faces - vertex_colors = np.array([[255,100,0,255], [0,100,255,255]])[vertex_label.astype(int)] - tm.Trimesh(faces=faces, vertices=vertices, vertex_colors=vertex_colors).show() + show_mesh(mesh, label=pred_class) if __name__ == '__main__': run_test() From a47524de48942376965ffabb3dfb845f5acfb25f Mon Sep 17 00:00:00 2001 From: ihahanov Date: Wed, 17 Nov 2021 14:16:35 +0200 Subject: [PATCH 06/67] fix visualization of mesh segmentation. add copying mesh before passing to the model --- data/segmentation_data.py | 1 + 1 file changed, 1 insertion(+) diff --git a/data/segmentation_data.py b/data/segmentation_data.py index e4f5a4f7..ee5db680 100644 --- a/data/segmentation_data.py +++ b/data/segmentation_data.py @@ -50,6 +50,7 @@ def __getitem__(self, index): mesh = Mesh(file=path, opt=self.opt, hold_history=True, export_folder=self.opt.export_folder) meta = {} meta['mesh'] = mesh + meta['path'] = path label = read_seg(self.seg_paths[index]) - self.offset label = pad(label, self.opt.ninput_edges, val=-1, dim=0) meta['label'] = label From 87eb6814aab40fea5f642fece9d24037d3a8c4e6 Mon Sep 17 00:00:00 2001 From: ihahanov Date: Mon, 22 Nov 2021 16:38:25 +0200 Subject: [PATCH 07/67] change main script --- data/blender_scripts/extract_vertex_labels.py | 5 +++-- vertex_label_to_edge.py | 6 +++--- visualize_annotations.py | 6 +++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/data/blender_scripts/extract_vertex_labels.py b/data/blender_scripts/extract_vertex_labels.py index 36eef2c5..fffef089 100644 --- a/data/blender_scripts/extract_vertex_labels.py +++ b/data/blender_scripts/extract_vertex_labels.py @@ -11,5 +11,6 @@ else: label.append(str(0)) -with open('/home/ihahanov/Projects/meshcnn/datasets/roof_seg/vseg/2620 Benefield Simplified + Annotated.eseg', 'w') as f: - f.write('\n'.join(label)) \ No newline at end of file +with open('/home/ihahanov/Projects/roof-measurements/dl_roof_extraction/meshcnn' + '/datasets/roof_seg/vseg/2510 Garrison Simplified + Annotated.eseg', 'w') as f: + f.write('\n'.join(label)) diff --git a/vertex_label_to_edge.py b/vertex_label_to_edge.py index 80413124..fb09f8c2 100644 --- a/vertex_label_to_edge.py +++ b/vertex_label_to_edge.py @@ -169,10 +169,10 @@ def show_mesh(edges, vs, label, colors=[[0,0,0,255], [120,120,120,255]]): def create_files(path): - print("path", glob.glob(os.path.join(path, 'train/*.obj'))) - for filename in glob.glob(os.path.join(path, 'train/*.obj')): + print("path", glob.glob(os.path.join(path, 'obj/*.obj'))) + for filename in glob.glob(os.path.join(path, 'obj/*.obj')): basename = os.path.splitext(os.path.basename(filename))[0] - v_label_name = os.path.join(os.path.join(path, 'seg'), basename + '_v.eseg') + v_label_name = os.path.join(os.path.join(path, 'vseg'), basename + '.eseg') label_name = os.path.join(os.path.join(path, 'seg'), basename + '.eseg') export_name_seseg = os.path.join(os.path.join(path, 'sseg'), basename + '.seseg') export_name_edges = os.path.join(os.path.join(path, 'edges'), basename + '.edges') diff --git a/visualize_annotations.py b/visualize_annotations.py index 955dee2d..4c2cbc05 100644 --- a/visualize_annotations.py +++ b/visualize_annotations.py @@ -5,7 +5,7 @@ import trimesh as tm import numpy as np from copy import deepcopy -from data.segmentation_data import show_mesh +from visualize import show_mesh import trimesh as tm @@ -29,9 +29,9 @@ def run_test(epoch=-1): opt.serial_batches = True # no shuffle dataset = DataLoader(opt) for i, data in enumerate(dataset): - mesh = deepcopy(data['mesh'][0]) + print(data['path']) - show_mesh(mesh, data['label'][0]) + show_mesh(data['mesh'][0], data['label']) if __name__ == '__main__': run_test() From 6e7571286d7b3a6a1cea0a932f49998d337b979e Mon Sep 17 00:00:00 2001 From: mrybakova Date: Tue, 23 Nov 2021 11:42:06 +0200 Subject: [PATCH 08/67] test metrics --- models/mesh_classifier.py | 14 ++++++++++ test.py | 54 +++++++++++++++++++++++++++++++++++---- 2 files changed, 63 insertions(+), 5 deletions(-) diff --git a/models/mesh_classifier.py b/models/mesh_classifier.py index 9ce50cb3..02841b33 100644 --- a/models/mesh_classifier.py +++ b/models/mesh_classifier.py @@ -1,4 +1,6 @@ import torch +import torchmetrics + from . import networks from os.path import join from util.util import seg_accuracy, print_network @@ -115,6 +117,18 @@ def test(self): correct = self.get_accuracy(pred_class, label_class) return correct, len(label_class) + def get_metrics(self, acc_metric, f1_metric, iou_metric): + with torch.no_grad(): + out = self.forward() + pred_class = out.data.max(1)[1] + label_class = self.labels + label_class[label_class == -1] = 0 + + acc = acc_metric(pred_class, label_class) + f1 = f1_metric(pred_class, label_class) + iou = iou_metric(pred_class, label_class) + return acc, f1, iou + def get_accuracy(self, pred, labels): """computes accuracy for classification / segmentation """ if self.opt.dataset_mode == 'classification': diff --git a/test.py b/test.py index ed6179c8..034576bc 100644 --- a/test.py +++ b/test.py @@ -1,13 +1,15 @@ +from copy import deepcopy + +import numpy as np +import torchmetrics +import trimesh as tm + from options.test_options import TestOptions from data import DataLoader from models import create_model from util.writer import Writer -import trimesh as tm -import numpy as np -from copy import deepcopy -import trimesh as tm def edges_to_path(edges, color=tm.visual.color.random_color()): lines = np.asarray(edges) args = tm.path.exchange.misc.lines_to_path(lines) @@ -54,5 +56,47 @@ def run_test(epoch=-1): return writer.acc +def run_test(epoch=-1): + print('Running Test') + opt = TestOptions().parse() + opt.serial_batches = True # no shuffle + dataset = DataLoader(opt) + model = create_model(opt) + writer = Writer(opt) + # test + writer.reset_counter() + + acc_metric = torchmetrics.Accuracy() + f1_metric = torchmetrics.F1(num_classes=2, mdmc_average='samplewise') + iou_metric = torchmetrics.IoU(2) + for i, data in enumerate(dataset): + mesh = deepcopy(data['mesh'][0]) + model.set_input(data) + + # pred_class = model.forward().max(1)[1] + # # show_mesh(mesh, pred_class[0]) + # edges = mesh.edges + # vertices = mesh.vs + # vertex_label = np.zeros(len(vertices)) + # for e_l, e in zip(pred_class[0], edges): + # if e_l == 1: + # vertex_label[e] = 1 + # faces = mesh.faces + # vertex_colors = np.array([tm.visual.random_color(), tm.visual.random_color()])[vertex_label.astype(int)] + # tm.Trimesh(faces=faces, vertices=vertices, vertex_colors=vertex_colors).show() + + acc, f1, iou = model.get_metrics(acc_metric, f1_metric, iou_metric) + # ncorrect, nexamples = model.test() + # + # writer.update_counter(ncorrect, nexamples) + print(f"Metrics on 3D model {i} - accuracy: {acc}, F1: {f1}, IoU: {iou}") + # writer.print_acc(epoch, writer.acc) + total_acc = acc_metric.compute() + total_f1 = f1_metric.compute() + total_iou = iou_metric.compute() + print(f'epoch: {epoch}, TEST ACC: {total_acc}, F1: {total_f1}, IoU: {total_iou}') + # return writer.acc + + if __name__ == '__main__': - run_test() + run_test() \ No newline at end of file From cbf240a0c43f3f517b7f2e2a87a500ee4782d980 Mon Sep 17 00:00:00 2001 From: ihahanov Date: Tue, 23 Nov 2021 16:04:36 +0200 Subject: [PATCH 09/67] add setting redis url from environment configuration --- models/mesh_classifier.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/models/mesh_classifier.py b/models/mesh_classifier.py index 02841b33..d0596a62 100644 --- a/models/mesh_classifier.py +++ b/models/mesh_classifier.py @@ -123,6 +123,8 @@ def get_metrics(self, acc_metric, f1_metric, iou_metric): pred_class = out.data.max(1)[1] label_class = self.labels label_class[label_class == -1] = 0 + pred_class = pred_class.to(self.device) + label_class = label_class.to(self.device) acc = acc_metric(pred_class, label_class) f1 = f1_metric(pred_class, label_class) From 1c49cfccb26aca8e3a98e484570b65ccf9808a3b Mon Sep 17 00:00:00 2001 From: mrybakova Date: Tue, 23 Nov 2021 17:16:33 +0200 Subject: [PATCH 10/67] train metrics --- test.py | 81 ++++++++++++++++++++++++++++++-------------------------- train.py | 8 +++--- 2 files changed, 49 insertions(+), 40 deletions(-) diff --git a/test.py b/test.py index 034576bc..f9c0d9d4 100644 --- a/test.py +++ b/test.py @@ -24,41 +24,41 @@ def show_edges(mesh, label, colors=[[0,0,0,255], [120,120,120,255]]): tm.Scene([edges_to_path(e, colors[int(l)]) for e, l in zip(edges, label)]).show() -def run_test(epoch=-1): +# def run_test(epoch=-1): +# print('Running Test') +# opt = TestOptions().parse() +# opt.serial_batches = True # no shuffle +# dataset = DataLoader(opt) +# model = create_model(opt) +# writer = Writer(opt) +# # test +# writer.reset_counter() +# for i, data in enumerate(dataset): +# mesh = deepcopy(data['mesh'][0]) +# model.set_input(data) +# +# # pred_class = model.forward().max(1)[1] +# # # show_mesh(mesh, pred_class[0]) +# # edges = mesh.edges +# # vertices = mesh.vs +# # vertex_label = np.zeros(len(vertices)) +# # for e_l, e in zip(pred_class[0], edges): +# # if e_l == 1: +# # vertex_label[e] = 1 +# # faces = mesh.faces +# # vertex_colors = np.array([tm.visual.random_color(), tm.visual.random_color()])[vertex_label.astype(int)] +# # tm.Trimesh(faces=faces, vertices=vertices, vertex_colors=vertex_colors).show() +# +# ncorrect, nexamples = model.test() +# +# writer.update_counter(ncorrect, nexamples) +# writer.print_acc(epoch, writer.acc) +# return writer.acc + + +def run_test(opt, epoch=-1, text='TEST'): print('Running Test') - opt = TestOptions().parse() - opt.serial_batches = True # no shuffle - dataset = DataLoader(opt) - model = create_model(opt) - writer = Writer(opt) - # test - writer.reset_counter() - for i, data in enumerate(dataset): - mesh = deepcopy(data['mesh'][0]) - model.set_input(data) - - # pred_class = model.forward().max(1)[1] - # # show_mesh(mesh, pred_class[0]) - # edges = mesh.edges - # vertices = mesh.vs - # vertex_label = np.zeros(len(vertices)) - # for e_l, e in zip(pred_class[0], edges): - # if e_l == 1: - # vertex_label[e] = 1 - # faces = mesh.faces - # vertex_colors = np.array([tm.visual.random_color(), tm.visual.random_color()])[vertex_label.astype(int)] - # tm.Trimesh(faces=faces, vertices=vertices, vertex_colors=vertex_colors).show() - - ncorrect, nexamples = model.test() - - writer.update_counter(ncorrect, nexamples) - writer.print_acc(epoch, writer.acc) - return writer.acc - - -def run_test(epoch=-1): - print('Running Test') - opt = TestOptions().parse() + # opt = TestOptions().parse() opt.serial_batches = True # no shuffle dataset = DataLoader(opt) model = create_model(opt) @@ -67,7 +67,7 @@ def run_test(epoch=-1): writer.reset_counter() acc_metric = torchmetrics.Accuracy() - f1_metric = torchmetrics.F1(num_classes=2, mdmc_average='samplewise') + f1_metric = torchmetrics.F1(num_classes=2, average='macro', mdmc_average='global') iou_metric = torchmetrics.IoU(2) for i, data in enumerate(dataset): mesh = deepcopy(data['mesh'][0]) @@ -94,9 +94,16 @@ def run_test(epoch=-1): total_acc = acc_metric.compute() total_f1 = f1_metric.compute() total_iou = iou_metric.compute() - print(f'epoch: {epoch}, TEST ACC: {total_acc}, F1: {total_f1}, IoU: {total_iou}') + print(f'epoch: {epoch}, {text} ACC: {total_acc}, F1: {total_f1}, IoU: {total_iou}') # return writer.acc + return total_acc, total_f1, total_iou +def run_test_on_test_data(epoch=-1): + opt = TestOptions().parse() + # dataset = DataLoader(opt) + total_acc, total_f1, total_iou = run_test(opt, epoch) + return total_acc, total_f1, total_iou if __name__ == '__main__': - run_test() \ No newline at end of file + # run_test() + run_test_on_test_data() \ No newline at end of file diff --git a/train.py b/train.py index ee778656..c33c73f3 100644 --- a/train.py +++ b/train.py @@ -3,7 +3,7 @@ from data import DataLoader from models import create_model from util.writer import Writer -from test import run_test +from test import run_test, run_test_on_test_data import torch @@ -57,8 +57,10 @@ writer.plot_model_wts(model, epoch) if epoch % opt.run_test_freq == 0: - acc = run_test(epoch) - writer.plot_acc(acc, epoch) + # acc = run_test(epoch) + run_test(opt, epoch, text='TRAIN') + run_test_on_test_data(epoch=epoch) + # writer.plot_acc(acc, epoch) torch.cuda.empty_cache() From 4f474d6cfd9a2edad6017a10262f829c42d753d3 Mon Sep 17 00:00:00 2001 From: ihahanov Date: Wed, 24 Nov 2021 10:24:51 +0200 Subject: [PATCH 11/67] add iou/acc metrics --- test.py | 113 ++++++++++++++++++++++++--------------------------- train.py | 5 ++- visualize.py | 3 ++ 3 files changed, 59 insertions(+), 62 deletions(-) diff --git a/test.py b/test.py index 034576bc..af72d05d 100644 --- a/test.py +++ b/test.py @@ -1,6 +1,7 @@ from copy import deepcopy import numpy as np +import torch import torchmetrics import trimesh as tm @@ -24,78 +25,70 @@ def show_edges(mesh, label, colors=[[0,0,0,255], [120,120,120,255]]): tm.Scene([edges_to_path(e, colors[int(l)]) for e, l in zip(edges, label)]).show() -def run_test(epoch=-1): +# def run_test(epoch=-1): +# print('Running Test') +# opt = TestOptions().parse() +# opt.serial_batches = True # no shuffle +# dataset = DataLoader(opt) +# model = create_model(opt) +# writer = Writer(opt) +# # test +# writer.reset_counter() +# for i, data in enumerate(dataset): +# mesh = deepcopy(data['mesh'][0]) +# model.set_input(data) +# +# # pred_class = model.forward().max(1)[1] +# # # show_mesh(mesh, pred_class[0]) +# # edges = mesh.edges +# # vertices = mesh.vs +# # vertex_label = np.zeros(len(vertices)) +# # for e_l, e in zip(pred_class[0], edges): +# # if e_l == 1: +# # vertex_label[e] = 1 +# # faces = mesh.faces +# # vertex_colors = np.array([tm.visual.random_color(), tm.visual.random_color()])[vertex_label.astype(int)] +# # tm.Trimesh(faces=faces, vertices=vertices, vertex_colors=vertex_colors).show() +# +# ncorrect, nexamples = model.test() +# +# writer.update_counter(ncorrect, nexamples) +# writer.print_acc(epoch, writer.acc) +# return writer.acc + + +def run_test(epoch=-1, data_phase='test'): print('Running Test') opt = TestOptions().parse() opt.serial_batches = True # no shuffle + opt.phase = data_phase dataset = DataLoader(opt) model = create_model(opt) writer = Writer(opt) # test writer.reset_counter() - for i, data in enumerate(dataset): - mesh = deepcopy(data['mesh'][0]) - model.set_input(data) - - # pred_class = model.forward().max(1)[1] - # # show_mesh(mesh, pred_class[0]) - # edges = mesh.edges - # vertices = mesh.vs - # vertex_label = np.zeros(len(vertices)) - # for e_l, e in zip(pred_class[0], edges): - # if e_l == 1: - # vertex_label[e] = 1 - # faces = mesh.faces - # vertex_colors = np.array([tm.visual.random_color(), tm.visual.random_color()])[vertex_label.astype(int)] - # tm.Trimesh(faces=faces, vertices=vertices, vertex_colors=vertex_colors).show() - ncorrect, nexamples = model.test() - - writer.update_counter(ncorrect, nexamples) - writer.print_acc(epoch, writer.acc) - return writer.acc - - -def run_test(epoch=-1): - print('Running Test') - opt = TestOptions().parse() - opt.serial_batches = True # no shuffle - dataset = DataLoader(opt) - model = create_model(opt) - writer = Writer(opt) - # test - writer.reset_counter() - - acc_metric = torchmetrics.Accuracy() - f1_metric = torchmetrics.F1(num_classes=2, mdmc_average='samplewise') - iou_metric = torchmetrics.IoU(2) - for i, data in enumerate(dataset): - mesh = deepcopy(data['mesh'][0]) - model.set_input(data) - - # pred_class = model.forward().max(1)[1] - # # show_mesh(mesh, pred_class[0]) - # edges = mesh.edges - # vertices = mesh.vs - # vertex_label = np.zeros(len(vertices)) - # for e_l, e in zip(pred_class[0], edges): - # if e_l == 1: - # vertex_label[e] = 1 - # faces = mesh.faces - # vertex_colors = np.array([tm.visual.random_color(), tm.visual.random_color()])[vertex_label.astype(int)] - # tm.Trimesh(faces=faces, vertices=vertices, vertex_colors=vertex_colors).show() - - acc, f1, iou = model.get_metrics(acc_metric, f1_metric, iou_metric) - # ncorrect, nexamples = model.test() - # - # writer.update_counter(ncorrect, nexamples) - print(f"Metrics on 3D model {i} - accuracy: {acc}, F1: {f1}, IoU: {iou}") + acc_metric = torchmetrics.Accuracy(num_classes=2, average='macro').to(model.device) + iou_metric = torchmetrics.IoU(num_classes=2).to(model.device) + with torch.no_grad(): + for i, data in enumerate(dataset): + model.set_input(data) + out = model.forward() + pred_class = out.data.max(1)[1] + label_class = model.labels + pred_class = pred_class.to(model.device) + label_class = label_class.to(model.device) + not_padding = label_class != -1 + label_class = label_class[not_padding] + pred_class = pred_class[not_padding] + + acc = acc_metric(pred_class, label_class) + iou = iou_metric(pred_class, label_class) + # print(f"Metrics on 3D model {i} - accuracy: {acc}, F1: {f1}, IoU: {iou}") # writer.print_acc(epoch, writer.acc) total_acc = acc_metric.compute() - total_f1 = f1_metric.compute() total_iou = iou_metric.compute() - print(f'epoch: {epoch}, TEST ACC: {total_acc}, F1: {total_f1}, IoU: {total_iou}') - # return writer.acc + print(f'epoch: {epoch}, {data_phase.upper()} ACC: {total_acc}, IoU: {total_iou}') if __name__ == '__main__': diff --git a/train.py b/train.py index ee778656..27e28a60 100644 --- a/train.py +++ b/train.py @@ -57,8 +57,9 @@ writer.plot_model_wts(model, epoch) if epoch % opt.run_test_freq == 0: - acc = run_test(epoch) - writer.plot_acc(acc, epoch) + run_test(epoch, 'train') + run_test(epoch, 'test') + # writer.plot_acc(acc, epoch) torch.cuda.empty_cache() diff --git a/visualize.py b/visualize.py index 23558b5e..d30ba4b2 100644 --- a/visualize.py +++ b/visualize.py @@ -7,6 +7,7 @@ from copy import deepcopy from data.segmentation_data import show_mesh import trimesh as tm +import torch def edges_to_path(edges, color=tm.visual.color.random_color()): @@ -44,6 +45,7 @@ def run_test(epoch=-1): # test writer.reset_counter() for i, data in enumerate(dataset): + torch.cuda.empty_cache() mesh = deepcopy(data['mesh'][0]) # show_mesh(mesh, data['label'][0]) @@ -52,6 +54,7 @@ def run_test(epoch=-1): pred_class = model.forward().max(1)[1] # show_mesh(mesh, pred_class[0]) show_mesh(mesh, label=pred_class) + torch.cuda.empty_cache() if __name__ == '__main__': run_test() From 67ce6c725ed6412e8765e78ac01e35049b00e551 Mon Sep 17 00:00:00 2001 From: ihahanov Date: Wed, 24 Nov 2021 12:21:20 +0200 Subject: [PATCH 12/67] make metrics configurable from list --- models/networks.py | 3 ++- test.py | 21 ++++++++++++++------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/models/networks.py b/models/networks.py index c2a13e2e..8f6fe4b1 100644 --- a/models/networks.py +++ b/models/networks.py @@ -7,6 +7,7 @@ import torch.nn.functional as F from models.layers.mesh_pool import MeshPool from models.layers.mesh_unpool import MeshUnpool +from .losses import ce_jaccard ############################################################################### @@ -114,7 +115,7 @@ def define_loss(opt): if opt.dataset_mode == 'classification': loss = torch.nn.CrossEntropyLoss() elif opt.dataset_mode == 'segmentation': - loss = torch.nn.CrossEntropyLoss(ignore_index=-1) + loss = torch.nn.CrossEntropyLoss(ignore_index=-1, weight=torch.tensor([0.5, 2])) return loss ############################################################################## diff --git a/test.py b/test.py index af72d05d..708e386d 100644 --- a/test.py +++ b/test.py @@ -68,8 +68,12 @@ def run_test(epoch=-1, data_phase='test'): # test writer.reset_counter() - acc_metric = torchmetrics.Accuracy(num_classes=2, average='macro').to(model.device) - iou_metric = torchmetrics.IoU(num_classes=2).to(model.device) + metrics = [ + torchmetrics.Accuracy(num_classes=2, average='macro').to(model.device), + torchmetrics.Accuracy(num_classes=2).to(model.device), + torchmetrics.IoU(num_classes=2).to(model.device), + torchmetrics.F1(num_classes=2, average='macro').to(model.device) + ] with torch.no_grad(): for i, data in enumerate(dataset): model.set_input(data) @@ -82,13 +86,16 @@ def run_test(epoch=-1, data_phase='test'): label_class = label_class[not_padding] pred_class = pred_class[not_padding] - acc = acc_metric(pred_class, label_class) - iou = iou_metric(pred_class, label_class) + for m in metrics: + m(pred_class, label_class) # print(f"Metrics on 3D model {i} - accuracy: {acc}, F1: {f1}, IoU: {iou}") # writer.print_acc(epoch, writer.acc) - total_acc = acc_metric.compute() - total_iou = iou_metric.compute() - print(f'epoch: {epoch}, {data_phase.upper()} ACC: {total_acc}, IoU: {total_iou}') + metric_vals = [] + for m in metrics: + m_name = str(m).split('(')[0] + metric_vals.append(f'{m_name}: {m.compute()}') + metrics_str = ' '.join(metric_vals) + print(f'epoch: {epoch}, {data_phase.upper()} {metrics_str}') if __name__ == '__main__': From cf18e47da2f7ab585fdfb8f3b39d04619a2ff485 Mon Sep 17 00:00:00 2001 From: mrybakova Date: Thu, 25 Nov 2021 17:16:38 +0200 Subject: [PATCH 13/67] losses --- models/losses.py | 187 ++++++++++++++++++++++++++++++++++++++ models/mesh_classifier.py | 2 +- models/networks.py | 5 +- 3 files changed, 191 insertions(+), 3 deletions(-) create mode 100644 models/losses.py diff --git a/models/losses.py b/models/losses.py new file mode 100644 index 00000000..614e408b --- /dev/null +++ b/models/losses.py @@ -0,0 +1,187 @@ +"""Common image segmentation losses. +""" + +import torch + +from torch.nn import functional as F + + +def bce_loss(true, logits, pos_weight=None): + """Computes the weighted binary cross-entropy loss. + + Args: + true: a tensor of shape [B, 1, H, W]. + logits: a tensor of shape [B, 1, H, W]. Corresponds to + the raw output or logits of the model. + pos_weight: a scalar representing the weight attributed + to the positive class. This is especially useful for + an imbalanced dataset. + + Returns: + bce_loss: the weighted binary cross-entropy loss. + """ + bce_loss = F.binary_cross_entropy_with_logits( + logits.float(), + true.float(), + pos_weight=pos_weight, + ) + return bce_loss + + +def ce_loss(true, logits, weights, ignore=255): + """Computes the weighted multi-class cross-entropy loss. + + Args: + true: a tensor of shape [B, 1, H, W]. + logits: a tensor of shape [B, C, H, W]. Corresponds to + the raw output or logits of the model. + weight: a tensor of shape [C,]. The weights attributed + to each class. + ignore: the class index to ignore. + + Returns: + ce_loss: the weighted multi-class cross-entropy loss. + """ + ce_loss = F.cross_entropy( + logits.float(), + true.long(), + ignore_index=ignore, + weight=weights, + ) + return ce_loss + + +def dice_loss(true, logits, eps=1e-7): + """Computes the Sørensen–Dice loss. + + Note that PyTorch optimizers minimize a loss. In this + case, we would like to maximize the dice loss so we + return the negated dice loss. + + Args: + true: a tensor of shape [B, 1, H, W]. + logits: a tensor of shape [B, C, H, W]. Corresponds to + the raw output or logits of the model. + eps: added to the denominator for numerical stability. + + Returns: + dice_loss: the Sørensen–Dice loss. + """ + num_classes = logits.shape[1] + if num_classes == 1: + true_1_hot = torch.eye(num_classes + 1)[true.squeeze(1)] + true_1_hot = true_1_hot.permute(0, 3, 1, 2).float() + true_1_hot_f = true_1_hot[:, 0:1, :, :] + true_1_hot_s = true_1_hot[:, 1:2, :, :] + true_1_hot = torch.cat([true_1_hot_s, true_1_hot_f], dim=1) + pos_prob = torch.sigmoid(logits) + neg_prob = 1 - pos_prob + probas = torch.cat([pos_prob, neg_prob], dim=1) + else: + true_1_hot = torch.eye(num_classes)[true.squeeze(1)] + true_1_hot = true_1_hot.permute(0, 3, 1, 2).float() + probas = F.softmax(logits, dim=1) + true_1_hot = true_1_hot.type(logits.type()) + dims = (0,) + tuple(range(2, true.ndimension())) + intersection = torch.sum(probas * true_1_hot, dims) + cardinality = torch.sum(probas + true_1_hot, dims) + dice_loss = (2. * intersection / (cardinality + eps)).mean() + return (1 - dice_loss) + + +def jaccard_loss(true, logits, eps=1e-7): + """Computes the Jaccard loss, a.k.a the IoU loss. + + Note that PyTorch optimizers minimize a loss. In this + case, we would like to maximize the jaccard loss so we + return the negated jaccard loss. + + Args: + true: a tensor of shape [B, H, W] or [B, 1, H, W]. + logits: a tensor of shape [B, C, H, W]. Corresponds to + the raw output or logits of the model. + eps: added to the denominator for numerical stability. + + Returns: + jacc_loss: the Jaccard loss. + """ + num_classes = logits.shape[1] + if num_classes == 1: + true_1_hot = torch.eye(num_classes + 1)[true.squeeze(1)] + true_1_hot = true_1_hot.permute(0, 3, 1, 2).float() + true_1_hot_f = true_1_hot[:, 0:1, :, :] + true_1_hot_s = true_1_hot[:, 1:2, :, :] + true_1_hot = torch.cat([true_1_hot_s, true_1_hot_f], dim=1) + pos_prob = torch.sigmoid(logits) + neg_prob = 1 - pos_prob + probas = torch.cat([pos_prob, neg_prob], dim=1) + else: + true_1_hot = torch.eye(num_classes)[true.squeeze(1)] + true_1_hot = true_1_hot.permute(0, 3, 1, 2).float() + probas = F.softmax(logits, dim=1) + true_1_hot = true_1_hot.type(logits.type()) + dims = (0,) + tuple(range(2, true.ndimension())) + intersection = torch.sum(probas * true_1_hot, dims) + cardinality = torch.sum(probas + true_1_hot, dims) + union = cardinality - intersection + jacc_loss = (intersection / (union + eps)).mean() + return (1 - jacc_loss) + + +def tversky_loss(true, logits, alpha, beta, eps=1e-7): + """Computes the Tversky loss [1]. + + Args: + true: a tensor of shape [B, H, W] or [B, 1, H, W]. + logits: a tensor of shape [B, C, H, W]. Corresponds to + the raw output or logits of the model. + alpha: controls the penalty for false positives. + beta: controls the penalty for false negatives. + eps: added to the denominator for numerical stability. + + Returns: + tversky_loss: the Tversky loss. + + Notes: + alpha = beta = 0.5 => dice coeff + alpha = beta = 1 => tanimoto coeff + alpha + beta = 1 => F beta coeff + + References: + [1]: https://arxiv.org/abs/1706.05721 + """ + num_classes = logits.shape[1] + if num_classes == 1: + true_1_hot = torch.eye(num_classes + 1)[true.squeeze(1)] + true_1_hot = true_1_hot.permute(0, 3, 1, 2).float() + true_1_hot_f = true_1_hot[:, 0:1, :, :] + true_1_hot_s = true_1_hot[:, 1:2, :, :] + true_1_hot = torch.cat([true_1_hot_s, true_1_hot_f], dim=1) + pos_prob = torch.sigmoid(logits) + neg_prob = 1 - pos_prob + probas = torch.cat([pos_prob, neg_prob], dim=1) + else: + true_1_hot = torch.eye(num_classes)[true.squeeze(1)] + true_1_hot = true_1_hot.permute(0, 3, 1, 2).float() + probas = F.softmax(logits, dim=1) + true_1_hot = true_1_hot.type(logits.type()) + dims = (0,) + tuple(range(2, true.ndimension())) + intersection = torch.sum(probas * true_1_hot, dims) + fps = torch.sum(probas * (1 - true_1_hot), dims) + fns = torch.sum((1 - probas) * true_1_hot, dims) + num = intersection + denom = intersection + (alpha * fps) + (beta * fns) + tversky_loss = (num / (denom + eps)).mean() + return (1 - tversky_loss) + + +def ce_dice(true, pred, log=False, w1=1, w2=1): + pass + + +def ce_jaccard(true, pred, log=False, w1=1, w2=1): + pass + + +def focal_loss(true, pred): + pass \ No newline at end of file diff --git a/models/mesh_classifier.py b/models/mesh_classifier.py index d0596a62..2b9bcc26 100644 --- a/models/mesh_classifier.py +++ b/models/mesh_classifier.py @@ -34,7 +34,7 @@ def __init__(self, opt): self.net = networks.define_classifier(opt.input_nc, opt.ncf, opt.ninput_edges, opt.nclasses, opt, self.gpu_ids, opt.arch, opt.init_type, opt.init_gain) self.net.train(self.is_train) - self.criterion = networks.define_loss(opt).to(self.device) + self.criterion = networks.define_loss(opt) if self.is_train: self.optimizer = torch.optim.Adam(self.net.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) diff --git a/models/networks.py b/models/networks.py index 8f6fe4b1..29011afd 100644 --- a/models/networks.py +++ b/models/networks.py @@ -7,7 +7,7 @@ import torch.nn.functional as F from models.layers.mesh_pool import MeshPool from models.layers.mesh_unpool import MeshUnpool -from .losses import ce_jaccard +from .losses import ce_jaccard, dice_loss ############################################################################### @@ -115,7 +115,8 @@ def define_loss(opt): if opt.dataset_mode == 'classification': loss = torch.nn.CrossEntropyLoss() elif opt.dataset_mode == 'segmentation': - loss = torch.nn.CrossEntropyLoss(ignore_index=-1, weight=torch.tensor([0.5, 2])) + # loss = torch.nn.CrossEntropyLoss(ignore_index=-1, weight=torch.tensor([0.5, 2])) + loss = lambda out, labels: dice_loss(labels.unsqueeze(1).unsqueeze(-1), out.unsqueeze(-1)) return loss ############################################################################## From e37d1bf26f29bdb1606553f77ec7c4bf4bb8ca5f Mon Sep 17 00:00:00 2001 From: mrybakova Date: Fri, 26 Nov 2021 18:44:48 +0200 Subject: [PATCH 14/67] remove padding --- models/mesh_classifier.py | 5 +++-- util/util.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/models/mesh_classifier.py b/models/mesh_classifier.py index 2b9bcc26..7007f08c 100644 --- a/models/mesh_classifier.py +++ b/models/mesh_classifier.py @@ -3,7 +3,7 @@ from . import networks from os.path import join -from util.util import seg_accuracy, print_network +from util.util import seg_accuracy, print_network, remove_padding class ClassifierModel: @@ -60,7 +60,8 @@ def forward(self): return out def backward(self, out): - self.loss = self.criterion(out, self.labels) + label_class, pred_class = remove_padding(self.labels, out) + self.loss = self.criterion(pred_class, label_class) self.loss.backward() def optimize_parameters(self): diff --git a/util/util.py b/util/util.py index 562c22f6..0767a21d 100644 --- a/util/util.py +++ b/util/util.py @@ -66,3 +66,15 @@ def calculate_entropy(np_array): entropy -= a * np.log(a) entropy /= np.log(np_array.shape[0]) return entropy + + +def remove_padding(label_class, pred_class): + not_padding = label_class != -1 + label_class = label_class[not_padding] + label_class = label_class.unsqueeze(0) + + not_padding = not_padding.repeat(2, 1) + not_padding = not_padding.unsqueeze(0) + pred_class = pred_class[not_padding] + pred_class = pred_class.reshape([1, 2, int(pred_class.size()[0] / 2)]) + return label_class, pred_class \ No newline at end of file From 2682e07b174a8bbb3714fa1fafca4a0a61510a1b Mon Sep 17 00:00:00 2001 From: mrybakova Date: Mon, 29 Nov 2021 13:08:53 +0200 Subject: [PATCH 15/67] combined loss --- models/networks.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/models/networks.py b/models/networks.py index 29011afd..9c8e9fea 100644 --- a/models/networks.py +++ b/models/networks.py @@ -7,7 +7,7 @@ import torch.nn.functional as F from models.layers.mesh_pool import MeshPool from models.layers.mesh_unpool import MeshUnpool -from .losses import ce_jaccard, dice_loss +from .losses import ce_jaccard, dice_loss, jaccard_loss, ce_loss, bce_loss ############################################################################### @@ -115,8 +115,10 @@ def define_loss(opt): if opt.dataset_mode == 'classification': loss = torch.nn.CrossEntropyLoss() elif opt.dataset_mode == 'segmentation': - # loss = torch.nn.CrossEntropyLoss(ignore_index=-1, weight=torch.tensor([0.5, 2])) - loss = lambda out, labels: dice_loss(labels.unsqueeze(1).unsqueeze(-1), out.unsqueeze(-1)) + loss_ce = torch.nn.CrossEntropyLoss(ignore_index=-1, weight=torch.tensor([0.5, 2])) + loss_dice = lambda out, labels: dice_loss(labels.unsqueeze(1).unsqueeze(-1), out.unsqueeze(-1)) + loss = lambda out, labels: loss_ce(out, labels) + loss_dice(out, labels) + # loss = lambda out, labels: ce_loss(labels.squeeze(), out.squeeze().transpose(0,1), weights=torch.FloatTensor([0.5, 2])) return loss ############################################################################## From 363a662ae99eaf90dfb274c6e3dd6dc93eeab4ee Mon Sep 17 00:00:00 2001 From: mrybakova Date: Mon, 29 Nov 2021 14:53:12 +0200 Subject: [PATCH 16/67] combined loss fixed --- models/networks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/models/networks.py b/models/networks.py index 9c8e9fea..b0a7d97d 100644 --- a/models/networks.py +++ b/models/networks.py @@ -115,10 +115,11 @@ def define_loss(opt): if opt.dataset_mode == 'classification': loss = torch.nn.CrossEntropyLoss() elif opt.dataset_mode == 'segmentation': - loss_ce = torch.nn.CrossEntropyLoss(ignore_index=-1, weight=torch.tensor([0.5, 2])) + # loss_ce = torch.nn.CrossEntropyLoss(ignore_index=-1, weight=torch.tensor([0.5, 2])) loss_dice = lambda out, labels: dice_loss(labels.unsqueeze(1).unsqueeze(-1), out.unsqueeze(-1)) + device = torch.device('cuda:{}'.format(opt.gpu_ids[0])) if opt.gpu_ids else torch.device('cpu') + loss_ce = lambda out, labels: ce_loss(labels.squeeze(), out.squeeze().transpose(0,1), weights=torch.FloatTensor([0.5, 2]).to(device)) loss = lambda out, labels: loss_ce(out, labels) + loss_dice(out, labels) - # loss = lambda out, labels: ce_loss(labels.squeeze(), out.squeeze().transpose(0,1), weights=torch.FloatTensor([0.5, 2])) return loss ############################################################################## From 31e89873adc0687ad3a79698487b6c522b09a908 Mon Sep 17 00:00:00 2001 From: ihahanov Date: Mon, 29 Nov 2021 15:02:38 +0200 Subject: [PATCH 17/67] add pytorch lightning train script. NOT TESTED --- models/losses.py | 175 ++++++++++++++++++++++++++++++++ models/mesh_classifier.py | 9 +- options/pl_options.py | 20 ++++ train_pl.py | 133 +++++++++++++++++++++++++ vertex_label_to_edge.py | 204 -------------------------------------- 5 files changed, 333 insertions(+), 208 deletions(-) create mode 100644 models/losses.py create mode 100644 options/pl_options.py create mode 100644 train_pl.py delete mode 100644 vertex_label_to_edge.py diff --git a/models/losses.py b/models/losses.py new file mode 100644 index 00000000..189d1bb3 --- /dev/null +++ b/models/losses.py @@ -0,0 +1,175 @@ +"""Common image segmentation losses. +""" + +import torch + +from torch.nn import functional as F + + +def bce_loss(true, logits, pos_weight=None): + """Computes the weighted binary cross-entropy loss. + Args: + true: a tensor of shape [B, 1, H, W]. + logits: a tensor of shape [B, 1, H, W]. Corresponds to + the raw output or logits of the model. + pos_weight: a scalar representing the weight attributed + to the positive class. This is especially useful for + an imbalanced dataset. + Returns: + bce_loss: the weighted binary cross-entropy loss. + """ + bce_loss = F.binary_cross_entropy_with_logits( + logits.float(), + true.float(), + pos_weight=pos_weight, + ) + return bce_loss + + +def ce_loss(true, logits, weights, ignore=255): + """Computes the weighted multi-class cross-entropy loss. + Args: + true: a tensor of shape [B, 1, H, W]. + logits: a tensor of shape [B, C, H, W]. Corresponds to + the raw output or logits of the model. + weight: a tensor of shape [C,]. The weights attributed + to each class. + ignore: the class index to ignore. + Returns: + ce_loss: the weighted multi-class cross-entropy loss. + """ + ce_loss = F.cross_entropy( + logits.float(), + true.long(), + ignore_index=ignore, + weight=weights, + ) + return ce_loss + + +def dice_loss(true, logits, eps=1e-7): + """Computes the Sørensen–Dice loss. + Note that PyTorch optimizers minimize a loss. In this + case, we would like to maximize the dice loss so we + return the negated dice loss. + Args: + true: a tensor of shape [B, 1, H, W]. + logits: a tensor of shape [B, C, H, W]. Corresponds to + the raw output or logits of the model. + eps: added to the denominator for numerical stability. + Returns: + dice_loss: the Sørensen–Dice loss. + """ + num_classes = logits.shape[1] + if num_classes == 1: + true_1_hot = torch.eye(num_classes + 1)[true.squeeze(1)] + true_1_hot = true_1_hot.permute(0, 3, 1, 2).float() + true_1_hot_f = true_1_hot[:, 0:1, :, :] + true_1_hot_s = true_1_hot[:, 1:2, :, :] + true_1_hot = torch.cat([true_1_hot_s, true_1_hot_f], dim=1) + pos_prob = torch.sigmoid(logits) + neg_prob = 1 - pos_prob + probas = torch.cat([pos_prob, neg_prob], dim=1) + else: + true_1_hot = torch.eye(num_classes)[true.squeeze(1)] + true_1_hot = true_1_hot.permute(0, 3, 1, 2).float() + probas = F.softmax(logits, dim=1) + true_1_hot = true_1_hot.type(logits.type()) + dims = (0,) + tuple(range(2, true.ndimension())) + intersection = torch.sum(probas * true_1_hot, dims) + cardinality = torch.sum(probas + true_1_hot, dims) + dice_loss = (2. * intersection / (cardinality + eps)).mean() + return (1 - dice_loss) + + +def jaccard_loss(true, logits, eps=1e-7): + """Computes the Jaccard loss, a.k.a the IoU loss. + Note that PyTorch optimizers minimize a loss. In this + case, we would like to maximize the jaccard loss so we + return the negated jaccard loss. + Args: + true: a tensor of shape [B, H, W] or [B, 1, H, W]. + logits: a tensor of shape [B, C, H, W]. Corresponds to + the raw output or logits of the model. + eps: added to the denominator for numerical stability. + Returns: + jacc_loss: the Jaccard loss. + """ + num_classes = logits.shape[1] + if num_classes == 1: + true_1_hot = torch.eye(num_classes + 1)[true.squeeze(1)] + true_1_hot = true_1_hot.permute(0, 3, 1, 2).float() + true_1_hot_f = true_1_hot[:, 0:1, :, :] + true_1_hot_s = true_1_hot[:, 1:2, :, :] + true_1_hot = torch.cat([true_1_hot_s, true_1_hot_f], dim=1) + pos_prob = torch.sigmoid(logits) + neg_prob = 1 - pos_prob + probas = torch.cat([pos_prob, neg_prob], dim=1) + else: + true_1_hot = torch.eye(num_classes)[true.squeeze(1)] + true_1_hot = true_1_hot.permute(0, 3, 1, 2).float() + probas = F.softmax(logits, dim=1) + true_1_hot = true_1_hot.type(logits.type()) + dims = (0,) + tuple(range(2, true.ndimension())) + intersection = torch.sum(probas * true_1_hot, dims) + cardinality = torch.sum(probas + true_1_hot, dims) + union = cardinality - intersection + jacc_loss = (intersection / (union + eps)).mean() + return (1 - jacc_loss) + + +def tversky_loss(true, logits, alpha, beta, eps=1e-7): + """Computes the Tversky loss [1]. + Args: + true: a tensor of shape [B, H, W] or [B, 1, H, W]. + logits: a tensor of shape [B, C, H, W]. Corresponds to + the raw output or logits of the model. + alpha: controls the penalty for false positives. + beta: controls the penalty for false negatives. + eps: added to the denominator for numerical stability. + Returns: + tversky_loss: the Tversky loss. + Notes: + alpha = beta = 0.5 => dice coeff + alpha = beta = 1 => tanimoto coeff + alpha + beta = 1 => F beta coeff + References: + [1]: https://arxiv.org/abs/1706.05721 + """ + num_classes = logits.shape[1] + if num_classes == 1: + true_1_hot = torch.eye(num_classes + 1)[true.squeeze(1)] + true_1_hot = true_1_hot.permute(0, 3, 1, 2).float() + true_1_hot_f = true_1_hot[:, 0:1, :, :] + true_1_hot_s = true_1_hot[:, 1:2, :, :] + true_1_hot = torch.cat([true_1_hot_s, true_1_hot_f], dim=1) + pos_prob = torch.sigmoid(logits) + neg_prob = 1 - pos_prob + probas = torch.cat([pos_prob, neg_prob], dim=1) + else: + true_1_hot = torch.eye(num_classes)[true.squeeze(1)] + true_1_hot = true_1_hot.permute(0, 3, 1, 2).float() + probas = F.softmax(logits, dim=1) + true_1_hot = true_1_hot.type(logits.type()) + dims = (0,) + tuple(range(2, true.ndimension())) + intersection = torch.sum(probas * true_1_hot, dims) + fps = torch.sum(probas * (1 - true_1_hot), dims) + fns = torch.sum((1 - probas) * true_1_hot, dims) + num = intersection + denom = intersection + (alpha * fps) + (beta * fns) + tversky_loss = (num / (denom + eps)).mean() + return (1 - tversky_loss) + + +def ce_dice(true, pred, log=False, w1=1, w2=1): + pass + + +def ce_jaccard(true, pred, weights=torch.tensor([0.5, 2])): + # return jaccard_loss(true, pred) + return ce_loss(true, pred, weights.to(pred.device), ignore=-1) + \ + jaccard_loss(true.unsqueeze(-1).unsqueeze(1), pred.unsqueeze(-1)) + + +def focal_loss(true, pred): + pass \ No newline at end of file diff --git a/models/mesh_classifier.py b/models/mesh_classifier.py index d0596a62..6d1ffaff 100644 --- a/models/mesh_classifier.py +++ b/models/mesh_classifier.py @@ -34,10 +34,12 @@ def __init__(self, opt): self.net = networks.define_classifier(opt.input_nc, opt.ncf, opt.ninput_edges, opt.nclasses, opt, self.gpu_ids, opt.arch, opt.init_type, opt.init_gain) self.net.train(self.is_train) - self.criterion = networks.define_loss(opt).to(self.device) + from .losses import ce_jaccard + self.criterion = ce_jaccard#networks.define_loss(opt).to(self.device) if self.is_train: - self.optimizer = torch.optim.Adam(self.net.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) + # self.optimizer = torch.optim.Adam(self.net.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) + self.optimizer = torch.optim.SGD(self.net.parameters(), lr=opt.lr, momentum=0.9, weight_decay=0.0001) self.scheduler = networks.get_scheduler(self.optimizer, opt) print_network(self.net) @@ -60,7 +62,7 @@ def forward(self): return out def backward(self, out): - self.loss = self.criterion(out, self.labels) + self.loss = self.criterion(self.labels, out) self.loss.backward() def optimize_parameters(self): @@ -87,7 +89,6 @@ def load_network(self, which_epoch): del state_dict._metadata net.load_state_dict(state_dict) - def save_network(self, which_epoch): """save model to disk""" save_filename = '%s_net.pth' % (which_epoch) diff --git a/options/pl_options.py b/options/pl_options.py new file mode 100644 index 00000000..f208fe9f --- /dev/null +++ b/options/pl_options.py @@ -0,0 +1,20 @@ +from .train_options import TrainOptions + +class PLOptions(TrainOptions): + def initialize(self): + TrainOptions.initialize(self) + parser.add_argument('--gpus', type=int, default=1) + parser.add_argument('--max_epochs', type=int, default=60) + parser.add_argument('--learning_rate', default=1e-3) + parser.add_argument('--max_image_size', default=128) + parser.add_argument('--num_classes', default=3) + parser.add_argument('--pretrained', default=True) + + parser.add_argument('--train_data', default='../../data/windows/set_1/train') + parser.add_argument('--test_data', default='../../data/windows/set_1/test') + parser.add_argument('--label_file', default='../../data/windows/labels.txt') + parser.add_argument('--train_augmentation', default=True) + + parser.add_argument('--progress_bar_refresh_rate', type=int, default=20) + parser.add_argument('--default_root_dir', default='../../models/test_classification/densenet161/', + help='pytorch-lightning log path') diff --git a/train_pl.py b/train_pl.py new file mode 100644 index 00000000..473a82a1 --- /dev/null +++ b/train_pl.py @@ -0,0 +1,133 @@ +import argparse +import os +import random + +import cv2 +import matplotlib.pyplot as plt +import torch +import pytorch_lightning as pl +from torch.utils.data import Dataset, DataLoader +import glob +from torchvision import transforms as T +from torchvision import transforms +import json +import numpy as np +import imgaug as ia +from utils.image_processing import resize_image, pad_image +import torchmodels +import torchmetrics +from utils.image_processing import enhance_contrast +from imgaug import augmenters as iaa +import pandas as pd +from options.pl_options import PLOptions + + +class MeshSegmenter(pl.LightningModule): + + def __init__(self, opt): + super().__init__() + self.opt = opt + self.model = create_model(opt) + self.train_metrics = [ + torchmetrics.Accuracy(num_classes=opt.nclasses, average='macro').to(model.device), + torchmetrics.IoU(num_classes=opt.nclasses).to(model.device), + torchmetrics.F1(num_classes=opt.nclasses, average='macro').to(model.device) + ] + self.val_metrics = [ + torchmetrics.Accuracy(num_classes=opt.nclasses, average='macro').to(model.device), + torchmetrics.IoU(num_classes=opt.nclasses).to(model.device), + torchmetrics.F1(num_classes=opt.nclasses, average='macro').to(model.device) + ] + + def training_step(self, batch, idx): + self.model.set_input(batch) + out = self.model.forward() + loss = self.criterion(self.model.labels, out) + + pred_class = out.data.max(1)[1] + not_padding = label_class != -1 + label_class = self.model.labels[not_padding] + pred_class = pred_class[not_padding] + + for m in self.train_metrics: + val = m(pred_class, label_class) + metric_name = str(m).split('(')[0] + self.log(metric_name.lower(), val, logger=True) + self.log('loss', loss) + return loss + + def validation_step(self, batch, idx): + self.model.set_input(batch) + out = self.model.forward() + loss = self.criterion(self.model.labels, out) + + pred_class = out.data.max(1)[1] + not_padding = label_class != -1 + label_class = self.model.labels[not_padding] + pred_class = pred_class[not_padding] + + for m in self.val_metrics: + val = m(pred_class, label_class) + metric_name = str(m).split('(')[0] + self.log('val_' + metric_name.lower(), val, logger=True) + self.log('val_loss', loss) + return loss + + def forward(self, image): + return self.model(image) + + def on_train_epoch_end(self, unused = None): + for m in self.train_metrics: + m.reset() + + def on_validation_epoch_end(self) -> None: + for m in self.val_metrics: + m.reset() + + def train_dataloader(self): + self.opt.phase = 'train' + return DataLoader(self.opt) + + def val_dataloader(self): + self.opt.phase = 'test' + return DataLoader(self.opt) + + def configure_optimizers(self): + opt = torch.optim.SGD(self.model.parameters(), lr=self.kwargs.get('learning_rate', 1e-3), + momentum=0.9, + weight_decay=0.0002) + sched = torch.optim.lr_scheduler.CosineAnnealingLR(opt, self.kwargs['max_epochs'] * 3) + return [opt], [sched] + + +def argument_parser(): + parser = argparse.ArgumentParser() + + parser.add_argument('--gpus', type=int, default=1) + parser.add_argument('--max_epochs', type=int, default=60) + parser.add_argument('--learning_rate', default=1e-3) + parser.add_argument('--max_image_size', default=128) + parser.add_argument('--num_classes', default=3) + parser.add_argument('--pretrained', default=True) + + parser.add_argument('--train_data', default='../../data/windows/set_1/train') + parser.add_argument('--test_data', default='../../data/windows/set_1/test') + parser.add_argument('--label_file', default='../../data/windows/labels.txt') + parser.add_argument('--train_augmentation', default=True) + + parser.add_argument('--progress_bar_refresh_rate', type=int, default=20) + parser.add_argument('--default_root_dir', default='../../models/test_classification/densenet161/', help='pytorch-lightning log path') + # parser.add_argument('--resume_from_checkpoint', default='../../models/test_classification/densenet121/lightning_logs/version_56/checkpoints/epoch=44-val_acc_epoch=0.98.ckpt') + return parser + + +if __name__ == '__main__': + from pytorch_lightning.callbacks import ModelCheckpoint + args = PLOptions().parse() + model = MeshSegmenter(args) + trainer = pl.Trainer.from_argparse_args(args, + callbacks=[ModelCheckpoint(monitor='val_iou', + mode='max', + save_top_k=3, + filename='{epoch:02d}-{val_acc_epoch:.2f}',)]) + trainer.fit(model) \ No newline at end of file diff --git a/vertex_label_to_edge.py b/vertex_label_to_edge.py deleted file mode 100644 index fb09f8c2..00000000 --- a/vertex_label_to_edge.py +++ /dev/null @@ -1,204 +0,0 @@ -import numpy as np -import os -import glob -import filecmp -import sys - - -''' -Creates esseg files for accuracy with smooth transitions between classes -Requires Objects and corresponding labels per edge -Author: Rana Hanocka / Lisa Schneider - -@input: - path where seg, sseg, train, test folders are placed - -@output: - esseg files for all objects - to run it from cmd line: - python create_sseg.py /home/user/MedMeshCNN/datasets/human_seg/ -''' - -def compute_face_normals_and_areas(vs, faces): - face_normals = np.cross(vs[faces[:, 1]] - vs[faces[:, 0]], - vs[faces[:, 2]] - vs[faces[:, 1]]) - face_areas = np.sqrt((face_normals ** 2).sum(axis=1)) - face_normals /= face_areas[:, np.newaxis] - assert (not np.any(face_areas[:, np.newaxis] == 0)), 'has zero area face' - face_areas *= 0.5 - return face_normals, face_areas - - -def remove_non_manifolds(vs, faces): - edges_set = set() - mask = np.ones(len(faces), dtype=bool) - _, face_areas = compute_face_normals_and_areas(vs, faces) - for face_id, face in enumerate(faces): - if face_areas[face_id] == 0: - mask[face_id] = False - continue - faces_edges = [] - is_manifold = False - for i in range(3): - cur_edge = (face[i], face[(i + 1) % 3]) - if cur_edge in edges_set: - is_manifold = True - break - else: - faces_edges.append(cur_edge) - if is_manifold: - mask[face_id] = False - else: - for idx, edge in enumerate(faces_edges): - edges_set.add(edge) - return faces[mask], face_areas[mask] - -def get_gemm_edges(faces, export_name_edges): - """ - gemm_edges: array (#E x 4) of the 4 one-ring neighbors for each edge - sides: array (#E x 4) indices (values of: 0,1,2,3) indicating where an edge is in the gemm_edge entry of the 4 neighboring edges - for example edge i -> gemm_edges[gemm_edges[i], sides[i]] == [i, i, i, i] - """ - edge_nb = [] - sides = [] - edge2key = dict() - edges = [] - edges_count = 0 - nb_count = [] - for face_id, face in enumerate(faces): - faces_edges = [] - for i in range(3): - cur_edge = (face[i], face[(i + 1) % 3]) - faces_edges.append(cur_edge) - for idx, edge in enumerate(faces_edges): - edge = tuple(sorted(list(edge))) - faces_edges[idx] = edge - if edge not in edge2key: - edge2key[edge] = edges_count - edges.append(list(edge)) - edge_nb.append([-1, -1, -1, -1]) - sides.append([-1, -1, -1, -1]) - nb_count.append(0) - edges_count += 1 - for idx, edge in enumerate(faces_edges): - edge_key = edge2key[edge] - edge_nb[edge_key][nb_count[edge_key]] = edge2key[faces_edges[(idx + 1) % 3]] - edge_nb[edge_key][nb_count[edge_key] + 1] = edge2key[faces_edges[(idx + 2) % 3]] - nb_count[edge_key] += 2 - for idx, edge in enumerate(faces_edges): - edge_key = edge2key[edge] - sides[edge_key][nb_count[edge_key] - 2] = nb_count[edge2key[faces_edges[(idx + 1) % 3]]] - 1 - sides[edge_key][nb_count[edge_key] - 1] = nb_count[edge2key[faces_edges[(idx + 2) % 3]]] - 2 - edges = np.array(edges, dtype=np.int32) - np.savetxt(export_name_edges, edges, fmt='%i') - return edge_nb, edges - - -def load_faces(path): - with open(path, 'r') as f: - for line in f: - inner_list = [vertices.strip() for vertices in line.split(',')] - gemm_edges.append(inner_list) - return gemm_edges - - -def load_labels(path): - with open(path, 'r') as f: - content = f.read().splitlines() - return content - -def create_sseg_file(gemms, labels, export_name_seseg): - gemmlabels = {} - classes = len(np.unique(labels)) - class_to_idx = {v: i for i, v in enumerate(np.unique(labels))} - totaledges = len(gemms) - sseg = np.zeros([ totaledges, classes]) - for i, edges in enumerate(gemms): - alllabels = [] - for edge in range(len(edges)): - lookupEdge = edges[edge] - label = labels[lookupEdge] - alllabels.append(label) - gemmlabels[i] = alllabels - - for i, edges in enumerate(gemms): - gemmlab = gemmlabels[i] - uniqueValues, counts = np.unique(gemmlab, return_counts=True) - for j, label in enumerate(uniqueValues): - weight = 0.125*counts[j] - sseg[i][class_to_idx[label]] = weight - np.savetxt(export_name_seseg, sseg, fmt='%1.6f') - -def get_obj(file): - vs, faces = [], [] - f = open(file) - for line in f: - line = line.strip() - splitted_line = line.split() - if not splitted_line: - continue - elif splitted_line[0] == 'v': - vs.append([float(v) for v in splitted_line[1:4]]) - elif splitted_line[0] == 'f': - face_vertex_ids = [int(c.split('/')[0]) for c in splitted_line[1:]] - assert len(face_vertex_ids) == 3 - face_vertex_ids = [(ind - 1) if (ind >= 0) else (len(vs) + ind) - for ind in face_vertex_ids] - faces.append(face_vertex_ids) - f.close() - vs = np.asarray(vs) - faces = np.asarray(faces, dtype=int) - assert np.logical_and(faces >= 0, faces < len(vs)).all() - return faces, vs - - -import trimesh as tm -def edges_to_path(edges, color=tm.visual.color.random_color()): - lines = np.asarray(edges) - args = tm.path.exchange.misc.lines_to_path(lines) - colors = [color for _ in range(len(args['entities']))] - path = tm.path.Path3D(**args, colors=colors) - return path - - -def show_mesh(edges, vs, label, colors=[[0,0,0,255], [120,120,120,255]]): - colors = np.array(colors) - edges = vs[edges] - tm.Scene([edges_to_path(e, colors[int(l)]) for e, l in zip(edges, label)]).show() - - - -def create_files(path): - print("path", glob.glob(os.path.join(path, 'obj/*.obj'))) - for filename in glob.glob(os.path.join(path, 'obj/*.obj')): - basename = os.path.splitext(os.path.basename(filename))[0] - v_label_name = os.path.join(os.path.join(path, 'vseg'), basename + '.eseg') - label_name = os.path.join(os.path.join(path, 'seg'), basename + '.eseg') - export_name_seseg = os.path.join(os.path.join(path, 'sseg'), basename + '.seseg') - export_name_edges = os.path.join(os.path.join(path, 'edges'), basename + '.edges') - - faces, vs = get_obj(filename) - faces, face_areas = remove_non_manifolds(vs, faces) - gemms, edges = get_gemm_edges(faces, export_name_edges) - with open(v_label_name) as f: - v_label = np.array(f.readlines(), dtype=int) - - edge_label = [] - for e in edges: - if v_label[e[0]] == 1 and v_label[e[1]] == 1: - edge_label.append(str(2)) - else: - edge_label.append(str(1)) - - with open(label_name, 'w') as f: - f.write('\n'.join(edge_label)) - - if os.path.isfile(label_name): - - create_sseg_file(gemms, edge_label, export_name_seseg) - else: - print(label_name, "is no directory") - - -if __name__ == '__main__': - create_files(sys.argv[1]) \ No newline at end of file From 78e230b4c7c322995ea652adfb10ee8dc700db9b Mon Sep 17 00:00:00 2001 From: ihahanov Date: Mon, 29 Nov 2021 15:30:52 +0200 Subject: [PATCH 18/67] test pytorch lightning script --- models/mesh_classifier.py | 6 ----- options/pl_options.py | 19 +++++--------- train_pl.py | 53 ++++++++++++++------------------------- 3 files changed, 25 insertions(+), 53 deletions(-) diff --git a/models/mesh_classifier.py b/models/mesh_classifier.py index 6d1ffaff..91a815fd 100644 --- a/models/mesh_classifier.py +++ b/models/mesh_classifier.py @@ -37,12 +37,6 @@ def __init__(self, opt): from .losses import ce_jaccard self.criterion = ce_jaccard#networks.define_loss(opt).to(self.device) - if self.is_train: - # self.optimizer = torch.optim.Adam(self.net.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) - self.optimizer = torch.optim.SGD(self.net.parameters(), lr=opt.lr, momentum=0.9, weight_decay=0.0001) - self.scheduler = networks.get_scheduler(self.optimizer, opt) - print_network(self.net) - if not self.is_train or opt.continue_train: self.load_network(opt.which_epoch) diff --git a/options/pl_options.py b/options/pl_options.py index f208fe9f..d8277e7a 100644 --- a/options/pl_options.py +++ b/options/pl_options.py @@ -3,18 +3,11 @@ class PLOptions(TrainOptions): def initialize(self): TrainOptions.initialize(self) - parser.add_argument('--gpus', type=int, default=1) - parser.add_argument('--max_epochs', type=int, default=60) - parser.add_argument('--learning_rate', default=1e-3) - parser.add_argument('--max_image_size', default=128) - parser.add_argument('--num_classes', default=3) - parser.add_argument('--pretrained', default=True) + self.parser.add_argument('--gpus', type=int, default=1) + self.parser.add_argument('--max_epochs', type=int, default=200) + self.parser.add_argument('--nclasses', type=int, default=2) + self.parser.add_argument('--input_nc', type=int, default=5) - parser.add_argument('--train_data', default='../../data/windows/set_1/train') - parser.add_argument('--test_data', default='../../data/windows/set_1/test') - parser.add_argument('--label_file', default='../../data/windows/labels.txt') - parser.add_argument('--train_augmentation', default=True) - - parser.add_argument('--progress_bar_refresh_rate', type=int, default=20) - parser.add_argument('--default_root_dir', default='../../models/test_classification/densenet161/', + self.parser.add_argument('--progress_bar_refresh_rate', type=int, default=20) + self.parser.add_argument('--default_root_dir', default='checkpoints/', help='pytorch-lightning log path') diff --git a/train_pl.py b/train_pl.py index 473a82a1..168c850b 100644 --- a/train_pl.py +++ b/train_pl.py @@ -2,24 +2,18 @@ import os import random -import cv2 import matplotlib.pyplot as plt import torch import pytorch_lightning as pl from torch.utils.data import Dataset, DataLoader import glob -from torchvision import transforms as T -from torchvision import transforms import json import numpy as np -import imgaug as ia -from utils.image_processing import resize_image, pad_image -import torchmodels import torchmetrics -from utils.image_processing import enhance_contrast -from imgaug import augmenters as iaa -import pandas as pd from options.pl_options import PLOptions +from data import DataLoader +from models import create_model +from models.losses import ce_jaccard class MeshSegmenter(pl.LightningModule): @@ -28,16 +22,17 @@ def __init__(self, opt): super().__init__() self.opt = opt self.model = create_model(opt) - self.train_metrics = [ - torchmetrics.Accuracy(num_classes=opt.nclasses, average='macro').to(model.device), - torchmetrics.IoU(num_classes=opt.nclasses).to(model.device), - torchmetrics.F1(num_classes=opt.nclasses, average='macro').to(model.device) - ] - self.val_metrics = [ - torchmetrics.Accuracy(num_classes=opt.nclasses, average='macro').to(model.device), - torchmetrics.IoU(num_classes=opt.nclasses).to(model.device), - torchmetrics.F1(num_classes=opt.nclasses, average='macro').to(model.device) - ] + self.criterion = ce_jaccard + self.train_metrics = torch.nn.ModuleList([ + torchmetrics.Accuracy(num_classes=opt.nclasses, average='macro'), + torchmetrics.IoU(num_classes=opt.nclasses), + torchmetrics.F1(num_classes=opt.nclasses, average='macro') + ]) + self.val_metrics = torch.nn.ModuleList([ + torchmetrics.Accuracy(num_classes=opt.nclasses, average='macro'), + torchmetrics.IoU(num_classes=opt.nclasses), + torchmetrics.F1(num_classes=opt.nclasses, average='macro') + ]) def training_step(self, batch, idx): self.model.set_input(batch) @@ -45,7 +40,7 @@ def training_step(self, batch, idx): loss = self.criterion(self.model.labels, out) pred_class = out.data.max(1)[1] - not_padding = label_class != -1 + not_padding = self.model.labels != -1 label_class = self.model.labels[not_padding] pred_class = pred_class[not_padding] @@ -62,7 +57,7 @@ def validation_step(self, batch, idx): loss = self.criterion(self.model.labels, out) pred_class = out.data.max(1)[1] - not_padding = label_class != -1 + not_padding = self.model.labels != -1 label_class = self.model.labels[not_padding] pred_class = pred_class[not_padding] @@ -93,10 +88,10 @@ def val_dataloader(self): return DataLoader(self.opt) def configure_optimizers(self): - opt = torch.optim.SGD(self.model.parameters(), lr=self.kwargs.get('learning_rate', 1e-3), + opt = torch.optim.SGD(self.model.net.parameters(), lr=self.opt.lr, momentum=0.9, weight_decay=0.0002) - sched = torch.optim.lr_scheduler.CosineAnnealingLR(opt, self.kwargs['max_epochs'] * 3) + sched = torch.optim.lr_scheduler.CosineAnnealingLR(opt, self.opt.max_epochs * 2) return [opt], [sched] @@ -105,19 +100,9 @@ def argument_parser(): parser.add_argument('--gpus', type=int, default=1) parser.add_argument('--max_epochs', type=int, default=60) - parser.add_argument('--learning_rate', default=1e-3) - parser.add_argument('--max_image_size', default=128) - parser.add_argument('--num_classes', default=3) - parser.add_argument('--pretrained', default=True) - - parser.add_argument('--train_data', default='../../data/windows/set_1/train') - parser.add_argument('--test_data', default='../../data/windows/set_1/test') - parser.add_argument('--label_file', default='../../data/windows/labels.txt') - parser.add_argument('--train_augmentation', default=True) parser.add_argument('--progress_bar_refresh_rate', type=int, default=20) - parser.add_argument('--default_root_dir', default='../../models/test_classification/densenet161/', help='pytorch-lightning log path') - # parser.add_argument('--resume_from_checkpoint', default='../../models/test_classification/densenet121/lightning_logs/version_56/checkpoints/epoch=44-val_acc_epoch=0.98.ckpt') + parser.add_argument('--default_root_dir', default='checkpoints/', help='pytorch-lightning log path') return parser From efaf4721e12a1b9fbf76e6b3da33a1d382384c47 Mon Sep 17 00:00:00 2001 From: ihahanov Date: Mon, 29 Nov 2021 15:42:30 +0200 Subject: [PATCH 19/67] add removing -1 when compute loss --- data/make_annotation_from_vertex_labels.py | 3 +-- models/losses.py | 12 +++++++++--- train_pl.py | 21 +++++++++++---------- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/data/make_annotation_from_vertex_labels.py b/data/make_annotation_from_vertex_labels.py index 9f1bcf32..a1bdae7c 100644 --- a/data/make_annotation_from_vertex_labels.py +++ b/data/make_annotation_from_vertex_labels.py @@ -161,8 +161,7 @@ def show_mesh(edges, vs, label, colors=[[0,0,0,255], [120,120,120,255]]): def create_files(path): - print("path", glob.glob(os.path.join(path, 'train/*.obj'))) - for filename in glob.glob(os.path.join(path, 'train/*.obj')): + for filename in glob.glob(os.path.join(path, 'obj/*.obj')): basename = os.path.splitext(os.path.basename(filename))[0] v_label_name = os.path.join(os.path.join(path, 'vseg'), basename + '.eseg') label_name = os.path.join(os.path.join(path, 'seg'), basename + '.eseg') diff --git a/models/losses.py b/models/losses.py index 189d1bb3..3eb2f1d7 100644 --- a/models/losses.py +++ b/models/losses.py @@ -166,9 +166,15 @@ def ce_dice(true, pred, log=False, w1=1, w2=1): def ce_jaccard(true, pred, weights=torch.tensor([0.5, 2])): - # return jaccard_loss(true, pred) - return ce_loss(true, pred, weights.to(pred.device), ignore=-1) + \ - jaccard_loss(true.unsqueeze(-1).unsqueeze(1), pred.unsqueeze(-1)) + num_classses = pred.shape[1] + true = true.view(-1) + pred = pred.view(num_classses, -1) + not_padding = true != -1 + true = true[not_padding] + pred = pred[:, not_padding] + + return ce_loss(true.unsqueeze(0), pred.unsqueeze(0), weights.to(pred.device), ignore=-1) + \ + jaccard_loss(true.view(1, 1, -1, 1), pred.view(1, num_classses, -1, 1)) def focal_loss(true, pred): diff --git a/train_pl.py b/train_pl.py index 168c850b..237324ed 100644 --- a/train_pl.py +++ b/train_pl.py @@ -23,16 +23,17 @@ def __init__(self, opt): self.opt = opt self.model = create_model(opt) self.criterion = ce_jaccard - self.train_metrics = torch.nn.ModuleList([ - torchmetrics.Accuracy(num_classes=opt.nclasses, average='macro'), - torchmetrics.IoU(num_classes=opt.nclasses), - torchmetrics.F1(num_classes=opt.nclasses, average='macro') - ]) - self.val_metrics = torch.nn.ModuleList([ - torchmetrics.Accuracy(num_classes=opt.nclasses, average='macro'), - torchmetrics.IoU(num_classes=opt.nclasses), - torchmetrics.F1(num_classes=opt.nclasses, average='macro') - ]) + if self.training: + self.train_metrics = torch.nn.ModuleList([ + torchmetrics.Accuracy(num_classes=opt.nclasses, average='macro'), + torchmetrics.IoU(num_classes=opt.nclasses), + torchmetrics.F1(num_classes=opt.nclasses, average='macro') + ]) + self.val_metrics = torch.nn.ModuleList([ + torchmetrics.Accuracy(num_classes=opt.nclasses, average='macro'), + torchmetrics.IoU(num_classes=opt.nclasses), + torchmetrics.F1(num_classes=opt.nclasses, average='macro') + ]) def training_step(self, batch, idx): self.model.set_input(batch) From 5d8ca93c034ce45b714205dec974ff2552627ef3 Mon Sep 17 00:00:00 2001 From: ihahanov Date: Tue, 30 Nov 2021 20:51:46 +0200 Subject: [PATCH 20/67] add train.sh --- models/losses.py | 5 ++++- models/mesh_classifier.py | 15 +++++++++++---- options/pl_options.py | 2 ++ scripts/roof_seg/train.sh | 17 +++++++++++++++++ train_pl.py | 29 +++++++++++------------------ 5 files changed, 45 insertions(+), 23 deletions(-) create mode 100644 scripts/roof_seg/train.sh diff --git a/models/losses.py b/models/losses.py index 3eb2f1d7..93a6c0a6 100644 --- a/models/losses.py +++ b/models/losses.py @@ -173,7 +173,10 @@ def ce_jaccard(true, pred, weights=torch.tensor([0.5, 2])): true = true[not_padding] pred = pred[:, not_padding] - return ce_loss(true.unsqueeze(0), pred.unsqueeze(0), weights.to(pred.device), ignore=-1) + \ + if weights is not None: + weights = torch.tensor(weights).to(pred.device) + + return ce_loss(true.unsqueeze(0), pred.unsqueeze(0), weights, ignore=-1) + \ jaccard_loss(true.view(1, 1, -1, 1), pred.view(1, num_classses, -1, 1)) diff --git a/models/mesh_classifier.py b/models/mesh_classifier.py index 91a815fd..ecf38183 100644 --- a/models/mesh_classifier.py +++ b/models/mesh_classifier.py @@ -72,16 +72,23 @@ def load_network(self, which_epoch): """load model from disk""" save_filename = '%s_net.pth' % which_epoch load_path = join(self.save_dir, save_filename) + self.load_weights(load_path) + + def load_weights(self, load_path): net = self.net if isinstance(net, torch.nn.DataParallel): net = net.module print('loading the model from %s' % load_path) # PyTorch newer than 0.4 (e.g., built from # GitHub source), you can remove str() on self.device - state_dict = torch.load(load_path, map_location=str(self.device)) - if hasattr(state_dict, '_metadata'): - del state_dict._metadata - net.load_state_dict(state_dict) + saved_dict = torch.load(load_path, map_location=str(self.device)) + if hasattr(saved_dict, '_metadata'): + del saved_dict._metadata + + current_dict = net.state_dict() + filtered_dict = {k: v for k, v in saved_dict.items() if saved_dict[k].shape == current_dict[k].shape} + current_dict.update(filtered_dict) + net.load_state_dict(current_dict) def save_network(self, which_epoch): """save model to disk""" diff --git a/options/pl_options.py b/options/pl_options.py index d8277e7a..14406472 100644 --- a/options/pl_options.py +++ b/options/pl_options.py @@ -7,6 +7,8 @@ def initialize(self): self.parser.add_argument('--max_epochs', type=int, default=200) self.parser.add_argument('--nclasses', type=int, default=2) self.parser.add_argument('--input_nc', type=int, default=5) + self.parser.add_argument('--class_weights', nargs='+', default=[0.5, 2], type=float) + self.parser.add_argument('--from_pretrained', type=str, default=None) self.parser.add_argument('--progress_bar_refresh_rate', type=int, default=20) self.parser.add_argument('--default_root_dir', default='checkpoints/', diff --git a/scripts/roof_seg/train.sh b/scripts/roof_seg/train.sh new file mode 100644 index 00000000..8de36a77 --- /dev/null +++ b/scripts/roof_seg/train.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +## run the training +python train_pl.py \ +--dataroot datasets/roof_seg \ +--name roof_seg \ +--arch meshunet \ +--dataset_mode segmentation \ +--ncf 32 64 128 256 \ +--ninput_edges 14000 \ +--pool_res 12000 10500 9000 \ +--resblocks 3 \ +--batch_size 1 \ +--lr 0.001 \ +--num_aug 20 \ +--slide_verts 0.2 +#--from_pretrained checkpoints/human_seg/latest_net.pth \ No newline at end of file diff --git a/train_pl.py b/train_pl.py index 237324ed..123ca703 100644 --- a/train_pl.py +++ b/train_pl.py @@ -2,7 +2,6 @@ import os import random -import matplotlib.pyplot as plt import torch import pytorch_lightning as pl from torch.utils.data import Dataset, DataLoader @@ -14,6 +13,8 @@ from data import DataLoader from models import create_model from models.losses import ce_jaccard +import warnings +warnings.filterwarnings("ignore") class MeshSegmenter(pl.LightningModule): @@ -22,6 +23,9 @@ def __init__(self, opt): super().__init__() self.opt = opt self.model = create_model(opt) + if opt.from_pretrained is not None: + print('Loaded pretrained weights:', opt.from_pretrained) + self.model.load_weights(opt.from_pretrained) self.criterion = ce_jaccard if self.training: self.train_metrics = torch.nn.ModuleList([ @@ -38,7 +42,7 @@ def __init__(self, opt): def training_step(self, batch, idx): self.model.set_input(batch) out = self.model.forward() - loss = self.criterion(self.model.labels, out) + loss = self.criterion(self.model.labels, out, self.opt.class_weights) pred_class = out.data.max(1)[1] not_padding = self.model.labels != -1 @@ -48,14 +52,14 @@ def training_step(self, batch, idx): for m in self.train_metrics: val = m(pred_class, label_class) metric_name = str(m).split('(')[0] - self.log(metric_name.lower(), val, logger=True) - self.log('loss', loss) + self.log(metric_name.lower(), val, logger=True, prog_bar=True, on_epoch=True) + self.log('loss', loss, on_epoch=True) return loss def validation_step(self, batch, idx): self.model.set_input(batch) out = self.model.forward() - loss = self.criterion(self.model.labels, out) + loss = self.criterion(self.model.labels, out, self.opt.class_weights) pred_class = out.data.max(1)[1] not_padding = self.model.labels != -1 @@ -65,8 +69,8 @@ def validation_step(self, batch, idx): for m in self.val_metrics: val = m(pred_class, label_class) metric_name = str(m).split('(')[0] - self.log('val_' + metric_name.lower(), val, logger=True) - self.log('val_loss', loss) + self.log('val_' + metric_name.lower(), val, logger=True, prog_bar=True, on_epoch=True) + self.log('val_loss', loss, on_epoch=True) return loss def forward(self, image): @@ -96,17 +100,6 @@ def configure_optimizers(self): return [opt], [sched] -def argument_parser(): - parser = argparse.ArgumentParser() - - parser.add_argument('--gpus', type=int, default=1) - parser.add_argument('--max_epochs', type=int, default=60) - - parser.add_argument('--progress_bar_refresh_rate', type=int, default=20) - parser.add_argument('--default_root_dir', default='checkpoints/', help='pytorch-lightning log path') - return parser - - if __name__ == '__main__': from pytorch_lightning.callbacks import ModelCheckpoint args = PLOptions().parse() From ea211e26b853ece6c7a478c4fffb1e046f835746 Mon Sep 17 00:00:00 2001 From: ihahanov Date: Wed, 1 Dec 2021 10:39:10 +0200 Subject: [PATCH 21/67] refactor losses --- models/losses.py | 29 +++++++++++++++++++---------- scripts/roof_seg/train.sh | 6 +++--- train_pl.py | 7 +++++-- 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/models/losses.py b/models/losses.py index 93a6c0a6..a0e9df5c 100644 --- a/models/losses.py +++ b/models/losses.py @@ -38,6 +38,9 @@ def ce_loss(true, logits, weights, ignore=255): Returns: ce_loss: the weighted multi-class cross-entropy loss. """ + true = true.squeeze(-1).squeeze(1) + logits = logits.squeeze(-1) + ce_loss = F.cross_entropy( logits.float(), true.long(), @@ -95,6 +98,7 @@ def jaccard_loss(true, logits, eps=1e-7): Returns: jacc_loss: the Jaccard loss. """ + num_classes = logits.shape[1] if num_classes == 1: true_1_hot = torch.eye(num_classes + 1)[true.squeeze(1)] @@ -166,19 +170,24 @@ def ce_dice(true, pred, log=False, w1=1, w2=1): def ce_jaccard(true, pred, weights=torch.tensor([0.5, 2])): - num_classses = pred.shape[1] - true = true.view(-1) - pred = pred.view(num_classses, -1) - not_padding = true != -1 - true = true[not_padding] - pred = pred[:, not_padding] - if weights is not None: weights = torch.tensor(weights).to(pred.device) - return ce_loss(true.unsqueeze(0), pred.unsqueeze(0), weights, ignore=-1) + \ - jaccard_loss(true.view(1, 1, -1, 1), pred.view(1, num_classses, -1, 1)) + return ce_loss(true, pred, weights) + \ + jaccard_loss(true, pred) def focal_loss(true, pred): - pass \ No newline at end of file + pass + + +def postprocess(true, pred): + num_classses = pred.shape[1] + true = true.view(-1) + pred = pred.view(num_classses, -1) + not_padding = true != -1 + true = true[not_padding] + pred = pred[:, not_padding] + true = true.view(1, 1, -1, 1) + pred = pred.view(1, num_classses, -1, 1) + return true, pred \ No newline at end of file diff --git a/scripts/roof_seg/train.sh b/scripts/roof_seg/train.sh index 8de36a77..30989325 100644 --- a/scripts/roof_seg/train.sh +++ b/scripts/roof_seg/train.sh @@ -9,9 +9,9 @@ python train_pl.py \ --ncf 32 64 128 256 \ --ninput_edges 14000 \ --pool_res 12000 10500 9000 \ ---resblocks 3 \ +--resblocks 2 \ --batch_size 1 \ ---lr 0.001 \ +--lr 0.01 \ --num_aug 20 \ --slide_verts 0.2 -#--from_pretrained checkpoints/human_seg/latest_net.pth \ No newline at end of file +#--from_pretrained checkpoints/coseg_aliens/latest_net.pth \ No newline at end of file diff --git a/train_pl.py b/train_pl.py index 123ca703..048568aa 100644 --- a/train_pl.py +++ b/train_pl.py @@ -12,6 +12,7 @@ from options.pl_options import PLOptions from data import DataLoader from models import create_model +from models.losses import postprocess from models.losses import ce_jaccard import warnings warnings.filterwarnings("ignore") @@ -42,7 +43,8 @@ def __init__(self, opt): def training_step(self, batch, idx): self.model.set_input(batch) out = self.model.forward() - loss = self.criterion(self.model.labels, out, self.opt.class_weights) + true, pred = postprocess(self.model.labels, out) + loss = self.criterion(true, pred, self.opt.class_weights) pred_class = out.data.max(1)[1] not_padding = self.model.labels != -1 @@ -59,7 +61,8 @@ def training_step(self, batch, idx): def validation_step(self, batch, idx): self.model.set_input(batch) out = self.model.forward() - loss = self.criterion(self.model.labels, out, self.opt.class_weights) + true, pred = postprocess(self.model.labels, out) + loss = self.criterion(true, pred, self.opt.class_weights) pred_class = out.data.max(1)[1] not_padding = self.model.labels != -1 From 0e3292589736b63cb3b9bf8fc17e3af704ed2cae Mon Sep 17 00:00:00 2001 From: mrybakova Date: Wed, 1 Dec 2021 14:13:26 +0200 Subject: [PATCH 22/67] losses update --- models/losses.py | 35 ++++++++++++++++++++++------------- models/networks.py | 17 +++++++++++++---- util/util.py | 11 +++++++---- 3 files changed, 42 insertions(+), 21 deletions(-) diff --git a/models/losses.py b/models/losses.py index 614e408b..ef217278 100644 --- a/models/losses.py +++ b/models/losses.py @@ -28,12 +28,12 @@ def bce_loss(true, logits, pos_weight=None): return bce_loss -def ce_loss(true, logits, weights, ignore=255): +def ce_loss(logits, true, weights=None, ignore=255): """Computes the weighted multi-class cross-entropy loss. Args: - true: a tensor of shape [B, 1, H, W]. - logits: a tensor of shape [B, C, H, W]. Corresponds to + true: a tensor of shape [1, N]. + logits: a tensor of shape [1, C, N]. Corresponds to the raw output or logits of the model. weight: a tensor of shape [C,]. The weights attributed to each class. @@ -42,6 +42,9 @@ def ce_loss(true, logits, weights, ignore=255): Returns: ce_loss: the weighted multi-class cross-entropy loss. """ + true = true.squeeze() + logits = logits.squeeze().transpose(0,1) + ce_loss = F.cross_entropy( logits.float(), true.long(), @@ -51,7 +54,7 @@ def ce_loss(true, logits, weights, ignore=255): return ce_loss -def dice_loss(true, logits, eps=1e-7): +def dice_loss(logits, true, eps=1e-7): """Computes the Sørensen–Dice loss. Note that PyTorch optimizers minimize a loss. In this @@ -59,14 +62,17 @@ def dice_loss(true, logits, eps=1e-7): return the negated dice loss. Args: - true: a tensor of shape [B, 1, H, W]. - logits: a tensor of shape [B, C, H, W]. Corresponds to + true: a tensor of shape [1, N]. + logits: a tensor of shape [1, C, N]. Corresponds to the raw output or logits of the model. eps: added to the denominator for numerical stability. Returns: dice_loss: the Sørensen–Dice loss. """ + true = true.unsqueeze(1).unsqueeze(-1) + logits = logits.unsqueeze(-1) + num_classes = logits.shape[1] if num_classes == 1: true_1_hot = torch.eye(num_classes + 1)[true.squeeze(1)] @@ -89,7 +95,7 @@ def dice_loss(true, logits, eps=1e-7): return (1 - dice_loss) -def jaccard_loss(true, logits, eps=1e-7): +def jaccard_loss(logits, true, eps=1e-7): """Computes the Jaccard loss, a.k.a the IoU loss. Note that PyTorch optimizers minimize a loss. In this @@ -97,14 +103,17 @@ def jaccard_loss(true, logits, eps=1e-7): return the negated jaccard loss. Args: - true: a tensor of shape [B, H, W] or [B, 1, H, W]. - logits: a tensor of shape [B, C, H, W]. Corresponds to + true: a tensor of shape [1, N]. + logits: a tensor of shape [1, C, N]. Corresponds to the raw output or logits of the model. eps: added to the denominator for numerical stability. Returns: jacc_loss: the Jaccard loss. """ + true = true.unsqueeze(1).unsqueeze(-1) + logits = logits.unsqueeze(-1) + num_classes = logits.shape[1] if num_classes == 1: true_1_hot = torch.eye(num_classes + 1)[true.squeeze(1)] @@ -175,12 +184,12 @@ def tversky_loss(true, logits, alpha, beta, eps=1e-7): return (1 - tversky_loss) -def ce_dice(true, pred, log=False, w1=1, w2=1): - pass +def ce_dice(logits, true, weights=None): + return ce_loss(logits, true, weights) + dice_loss(logits, true) -def ce_jaccard(true, pred, log=False, w1=1, w2=1): - pass +def ce_jaccard(logits, true, weights=None): + return ce_loss(logits, true, weights) + jaccard_loss(logits, true) def focal_loss(true, pred): diff --git a/models/networks.py b/models/networks.py index b0a7d97d..f5779240 100644 --- a/models/networks.py +++ b/models/networks.py @@ -7,7 +7,7 @@ import torch.nn.functional as F from models.layers.mesh_pool import MeshPool from models.layers.mesh_unpool import MeshUnpool -from .losses import ce_jaccard, dice_loss, jaccard_loss, ce_loss, bce_loss +from .losses import ce_jaccard, dice_loss, jaccard_loss, ce_loss, ce_dice ############################################################################### @@ -116,10 +116,19 @@ def define_loss(opt): loss = torch.nn.CrossEntropyLoss() elif opt.dataset_mode == 'segmentation': # loss_ce = torch.nn.CrossEntropyLoss(ignore_index=-1, weight=torch.tensor([0.5, 2])) - loss_dice = lambda out, labels: dice_loss(labels.unsqueeze(1).unsqueeze(-1), out.unsqueeze(-1)) + + # loss_dice = dice_loss + # loss_jaccard = jaccard_loss + device = torch.device('cuda:{}'.format(opt.gpu_ids[0])) if opt.gpu_ids else torch.device('cpu') - loss_ce = lambda out, labels: ce_loss(labels.squeeze(), out.squeeze().transpose(0,1), weights=torch.FloatTensor([0.5, 2]).to(device)) - loss = lambda out, labels: loss_ce(out, labels) + loss_dice(out, labels) + weights = torch.FloatTensor([0.5, 2]).to(device) + + # loss_ce = functools.partial(ce_loss, weights=weights) + loss_ce_dice = functools.partial(ce_dice, weights=weights) + # loss_ce_jaccard = functools.partial(ce_jaccard, weights=weights) + + loss = loss_ce_dice + # ToDo: loss option return loss ############################################################################## diff --git a/util/util.py b/util/util.py index 0767a21d..db42c733 100644 --- a/util/util.py +++ b/util/util.py @@ -69,12 +69,15 @@ def calculate_entropy(np_array): def remove_padding(label_class, pred_class): + num_classes = pred_class.size()[1] + label_class, pred_class = label_class.flatten(), pred_class.flatten() + not_padding = label_class != -1 label_class = label_class[not_padding] - label_class = label_class.unsqueeze(0) + label_class = label_class.view(1, -1) - not_padding = not_padding.repeat(2, 1) - not_padding = not_padding.unsqueeze(0) + not_padding = not_padding.repeat(num_classes) pred_class = pred_class[not_padding] - pred_class = pred_class.reshape([1, 2, int(pred_class.size()[0] / 2)]) + pred_class = pred_class.view(1, num_classes, -1) + return label_class, pred_class \ No newline at end of file From ba41de320af8b2dc0d46624aa957b8fcdf0f2510 Mon Sep 17 00:00:00 2001 From: mrybakova Date: Wed, 1 Dec 2021 18:59:17 +0200 Subject: [PATCH 23/67] loss option --- models/networks.py | 20 ++++++++++---------- options/base_options.py | 5 +++++ 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/models/networks.py b/models/networks.py index f5779240..7430ba36 100644 --- a/models/networks.py +++ b/models/networks.py @@ -115,20 +115,20 @@ def define_loss(opt): if opt.dataset_mode == 'classification': loss = torch.nn.CrossEntropyLoss() elif opt.dataset_mode == 'segmentation': - # loss_ce = torch.nn.CrossEntropyLoss(ignore_index=-1, weight=torch.tensor([0.5, 2])) - - # loss_dice = dice_loss - # loss_jaccard = jaccard_loss + # loss = torch.nn.CrossEntropyLoss(ignore_index=-1, weight=torch.tensor([0.5, 2])) device = torch.device('cuda:{}'.format(opt.gpu_ids[0])) if opt.gpu_ids else torch.device('cpu') - weights = torch.FloatTensor([0.5, 2]).to(device) + weights = torch.FloatTensor(opt.loss_weights).to(device) - # loss_ce = functools.partial(ce_loss, weights=weights) - loss_ce_dice = functools.partial(ce_dice, weights=weights) - # loss_ce_jaccard = functools.partial(ce_jaccard, weights=weights) + losses = { + 'ce': functools.partial(ce_loss, weights=weights), + 'dice': dice_loss, + 'jaccard': jaccard_loss, + 'ce_dice': functools.partial(ce_dice, weights=weights), + 'ce_jaccard': functools.partial(ce_jaccard, weights=weights) + } - loss = loss_ce_dice - # ToDo: loss option + loss = losses.get(opt.loss) return loss ############################################################################## diff --git a/options/base_options.py b/options/base_options.py index 09b9aa26..1364a6b1 100644 --- a/options/base_options.py +++ b/options/base_options.py @@ -26,6 +26,11 @@ def initialize(self): self.parser.add_argument('--num_groups', type=int, default=16, help='# of groups for groupnorm') self.parser.add_argument('--init_type', type=str, default='normal', help='network initialization [normal|xavier|kaiming|orthogonal]') self.parser.add_argument('--init_gain', type=float, default=0.02, help='scaling factor for normal, xavier and orthogonal.') + self.parser.add_argument('--loss', type=str, default='ce_dice', + help='loss function; possible values: ce, dice, jaccard, ce_dice, ce_jaccard') + self.parser.add_argument('--loss_weights', nargs='+', default=[0.5, 2], type=float, + help='weights for loss function, used only with ce/ce_dice/ce_jaccard losses') + # general params self.parser.add_argument('--num_threads', default=3, type=int, help='# threads for loading data') self.parser.add_argument('--gpu_ids', type=str, default='0', help='gpu ids: e.g. 0 0,1,2, 0,2. use -1 for CPU') From 3d2513db433bd0a143c0f04f96a97aaa4b59317e Mon Sep 17 00:00:00 2001 From: ihahanov Date: Thu, 2 Dec 2021 12:41:20 +0200 Subject: [PATCH 24/67] revert dice loss --- models/losses.py | 8 ++++++-- models/networks.py | 2 -- train_pl.py | 40 ++++++++++++++-------------------------- 3 files changed, 20 insertions(+), 30 deletions(-) diff --git a/models/losses.py b/models/losses.py index a0e9df5c..8b271778 100644 --- a/models/losses.py +++ b/models/losses.py @@ -165,8 +165,12 @@ def tversky_loss(true, logits, alpha, beta, eps=1e-7): return (1 - tversky_loss) -def ce_dice(true, pred, log=False, w1=1, w2=1): - pass +def ce_dice(true, pred, weights=torch.tensor([0.5, 2])): + if weights is not None: + weights = torch.tensor(weights).to(pred.device) + + return ce_loss(true, pred, weights) + \ + dice_loss(true, pred) def ce_jaccard(true, pred, weights=torch.tensor([0.5, 2])): diff --git a/models/networks.py b/models/networks.py index 7430ba36..816baa46 100644 --- a/models/networks.py +++ b/models/networks.py @@ -115,8 +115,6 @@ def define_loss(opt): if opt.dataset_mode == 'classification': loss = torch.nn.CrossEntropyLoss() elif opt.dataset_mode == 'segmentation': - # loss = torch.nn.CrossEntropyLoss(ignore_index=-1, weight=torch.tensor([0.5, 2])) - device = torch.device('cuda:{}'.format(opt.gpu_ids[0])) if opt.gpu_ids else torch.device('cpu') weights = torch.FloatTensor(opt.loss_weights).to(device) diff --git a/train_pl.py b/train_pl.py index 048568aa..eda93bdc 100644 --- a/train_pl.py +++ b/train_pl.py @@ -27,7 +27,7 @@ def __init__(self, opt): if opt.from_pretrained is not None: print('Loaded pretrained weights:', opt.from_pretrained) self.model.load_weights(opt.from_pretrained) - self.criterion = ce_jaccard + self.criterion = self.model.criterion if self.training: self.train_metrics = torch.nn.ModuleList([ torchmetrics.Accuracy(num_classes=opt.nclasses, average='macro'), @@ -40,46 +40,34 @@ def __init__(self, opt): torchmetrics.F1(num_classes=opt.nclasses, average='macro') ]) - def training_step(self, batch, idx): + def step(self, batch, is_train=True): self.model.set_input(batch) out = self.model.forward() true, pred = postprocess(self.model.labels, out) - loss = self.criterion(true, pred, self.opt.class_weights) + loss = self.criterion(true, pred) - pred_class = out.data.max(1)[1] - not_padding = self.model.labels != -1 - label_class = self.model.labels[not_padding] - pred_class = pred_class[not_padding] + true = true.view(-1) + pred = pred.argmax(1).view(-1) + prefix = '' if is_train else 'val_' for m in self.train_metrics: - val = m(pred_class, label_class) + val = m(pred, true) metric_name = str(m).split('(')[0] - self.log(metric_name.lower(), val, logger=True, prog_bar=True, on_epoch=True) - self.log('loss', loss, on_epoch=True) + self.log(prefix + metric_name.lower(), val, logger=True, prog_bar=True, on_epoch=True) + self.log(prefix + 'loss', loss, on_epoch=True) return loss - def validation_step(self, batch, idx): - self.model.set_input(batch) - out = self.model.forward() - true, pred = postprocess(self.model.labels, out) - loss = self.criterion(true, pred, self.opt.class_weights) + def training_step(self, batch, idx): - pred_class = out.data.max(1)[1] - not_padding = self.model.labels != -1 - label_class = self.model.labels[not_padding] - pred_class = pred_class[not_padding] + return self.step(batch, is_train=True) - for m in self.val_metrics: - val = m(pred_class, label_class) - metric_name = str(m).split('(')[0] - self.log('val_' + metric_name.lower(), val, logger=True, prog_bar=True, on_epoch=True) - self.log('val_loss', loss, on_epoch=True) - return loss + def validation_step(self, batch, idx): + return self.step(batch, is_train=False) def forward(self, image): return self.model(image) - def on_train_epoch_end(self, unused = None): + def on_train_epoch_end(self, unused=None): for m in self.train_metrics: m.reset() From 35bd1d7a513d5f2d852957f022d3d087f5dc6871 Mon Sep 17 00:00:00 2001 From: ihahanov Date: Thu, 2 Dec 2021 14:44:00 +0200 Subject: [PATCH 25/67] init dvc --- .dvc/.gitignore | 3 +++ .dvc/config | 0 .dvcignore | 3 +++ 3 files changed, 6 insertions(+) create mode 100644 .dvc/.gitignore create mode 100644 .dvc/config create mode 100644 .dvcignore diff --git a/.dvc/.gitignore b/.dvc/.gitignore new file mode 100644 index 00000000..528f30c7 --- /dev/null +++ b/.dvc/.gitignore @@ -0,0 +1,3 @@ +/config.local +/tmp +/cache diff --git a/.dvc/config b/.dvc/config new file mode 100644 index 00000000..e69de29b diff --git a/.dvcignore b/.dvcignore new file mode 100644 index 00000000..51973055 --- /dev/null +++ b/.dvcignore @@ -0,0 +1,3 @@ +# Add patterns of files dvc should ignore, which could improve +# the performance. Learn more at +# https://dvc.org/doc/user-guide/dvcignore From 4056e32ce8c7ed9db866df38c1866247c153643a Mon Sep 17 00:00:00 2001 From: ihahanov Date: Thu, 2 Dec 2021 14:57:39 +0200 Subject: [PATCH 26/67] add datasets to dvc --- .gitignore | 3 ++- datasets.dvc | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 datasets.dvc diff --git a/.gitignore b/.gitignore index df124a5c..78aa9831 100755 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,6 @@ # data files *.obj checkpoints -datasets +#datasets runs +/datasets diff --git a/datasets.dvc b/datasets.dvc new file mode 100644 index 00000000..5dc100cd --- /dev/null +++ b/datasets.dvc @@ -0,0 +1,5 @@ +outs: +- md5: 9662966745fe990e470452276e0701d1.dir + size: 184013105 + nfiles: 234 + path: datasets From fbece190e8d03ed2ef6f510e32ebe7cff7b4a0aa Mon Sep 17 00:00:00 2001 From: ihahanov Date: Thu, 2 Dec 2021 15:28:10 +0200 Subject: [PATCH 27/67] add only roof_seg dataset to index --- .gitignore | 1 - datasets.dvc | 5 ----- datasets/.gitignore | 1 + datasets/roof_seg.dvc | 5 +++++ 4 files changed, 6 insertions(+), 6 deletions(-) delete mode 100644 datasets.dvc create mode 100644 datasets/.gitignore create mode 100644 datasets/roof_seg.dvc diff --git a/.gitignore b/.gitignore index 78aa9831..dbe25587 100755 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,3 @@ checkpoints #datasets runs -/datasets diff --git a/datasets.dvc b/datasets.dvc deleted file mode 100644 index 5dc100cd..00000000 --- a/datasets.dvc +++ /dev/null @@ -1,5 +0,0 @@ -outs: -- md5: 9662966745fe990e470452276e0701d1.dir - size: 184013105 - nfiles: 234 - path: datasets diff --git a/datasets/.gitignore b/datasets/.gitignore new file mode 100644 index 00000000..dc61639c --- /dev/null +++ b/datasets/.gitignore @@ -0,0 +1 @@ +/roof_seg diff --git a/datasets/roof_seg.dvc b/datasets/roof_seg.dvc new file mode 100644 index 00000000..9aa0874b --- /dev/null +++ b/datasets/roof_seg.dvc @@ -0,0 +1,5 @@ +outs: +- md5: 16793820fa6e8cdb59dce3e175dd3a7d.dir + size: 184013105 + nfiles: 234 + path: roof_seg From 489cc13a2c0713ab3d850c701485ed57670c9ac9 Mon Sep 17 00:00:00 2001 From: ihahanov Date: Thu, 2 Dec 2021 15:47:03 +0200 Subject: [PATCH 28/67] add dvc remote on s3 bucket --- .dvc/config | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.dvc/config b/.dvc/config index e69de29b..5b81ce5f 100644 --- a/.dvc/config +++ b/.dvc/config @@ -0,0 +1,4 @@ +[core] + remote = storage +['remote "storage"'] + url = s3://machinelearning-assets/roofmeasurements/datasets/roof_segmentation From 79e9425b47c080f5a8ae6df1a4cdfb3f7122c69d Mon Sep 17 00:00:00 2001 From: ihahanov Date: Fri, 3 Dec 2021 14:07:16 +0200 Subject: [PATCH 29/67] add cardiff model to train set --- datasets/roof_seg.dvc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datasets/roof_seg.dvc b/datasets/roof_seg.dvc index 9aa0874b..5d6e329b 100644 --- a/datasets/roof_seg.dvc +++ b/datasets/roof_seg.dvc @@ -1,5 +1,5 @@ outs: -- md5: 16793820fa6e8cdb59dce3e175dd3a7d.dir - size: 184013105 - nfiles: 234 +- md5: 98089000c8f377aeff64977d234c8ffb.dir + size: 163643527 + nfiles: 209 path: roof_seg From 54ff2f6da405c6e71de5c2b7088b8c1d066bb711 Mon Sep 17 00:00:00 2001 From: ihahanov Date: Thu, 9 Dec 2021 12:48:46 +0200 Subject: [PATCH 30/67] fix metric for validation and step --- train_pl.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/train_pl.py b/train_pl.py index eda93bdc..c3ba6dc4 100644 --- a/train_pl.py +++ b/train_pl.py @@ -40,7 +40,7 @@ def __init__(self, opt): torchmetrics.F1(num_classes=opt.nclasses, average='macro') ]) - def step(self, batch, is_train=True): + def step(self, batch, metrics, metric_prefix=''): self.model.set_input(batch) out = self.model.forward() true, pred = postprocess(self.model.labels, out) @@ -49,8 +49,8 @@ def step(self, batch, is_train=True): true = true.view(-1) pred = pred.argmax(1).view(-1) - prefix = '' if is_train else 'val_' - for m in self.train_metrics: + prefix = metric_prefix + for m in metrics: val = m(pred, true) metric_name = str(m).split('(')[0] self.log(prefix + metric_name.lower(), val, logger=True, prog_bar=True, on_epoch=True) @@ -59,10 +59,10 @@ def step(self, batch, is_train=True): def training_step(self, batch, idx): - return self.step(batch, is_train=True) + return self.step(batch, self.train_metrics) def validation_step(self, batch, idx): - return self.step(batch, is_train=False) + return self.step(batch, self.val_metrics, metric_prefix='val') def forward(self, image): return self.model(image) From e0b6b7d36f4560249c263fd9f7c3065959f3e72b Mon Sep 17 00:00:00 2001 From: ihahanov Date: Thu, 9 Dec 2021 13:04:12 +0200 Subject: [PATCH 31/67] add underscore for val metrics --- train_pl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train_pl.py b/train_pl.py index c3ba6dc4..4520b8f1 100644 --- a/train_pl.py +++ b/train_pl.py @@ -62,7 +62,7 @@ def training_step(self, batch, idx): return self.step(batch, self.train_metrics) def validation_step(self, batch, idx): - return self.step(batch, self.val_metrics, metric_prefix='val') + return self.step(batch, self.val_metrics, metric_prefix='val_') def forward(self, image): return self.model(image) From edc51cd9f0274244576b62984079086bfda1ebf4 Mon Sep 17 00:00:00 2001 From: mrybakova Date: Thu, 9 Dec 2021 15:54:15 +0200 Subject: [PATCH 32/67] tuning --- tuning.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 tuning.py diff --git a/tuning.py b/tuning.py new file mode 100644 index 00000000..9d838ff7 --- /dev/null +++ b/tuning.py @@ -0,0 +1,37 @@ +from pytorch_lightning.callbacks import ModelCheckpoint +from pytorch_lightning import Trainer +from train_pl import * +from ray import tune +from ray.tune.integration.pytorch_lightning import TuneReportCallback + + +args = PLOptions().parse() + +def train_segmentation(config): + # args = PLOptions().parse() + args.num_aug = config.get('num_aug') + model = MeshSegmenter(args) + callback_tune = TuneReportCallback(metrics='val_iou', on="validation_end") + callback_lightning = ModelCheckpoint(monitor='val_iou', mode='max', save_top_k=3, + filename='{epoch:02d}-{val_acc_epoch:.2f}', ) + trainer = Trainer.from_argparse_args(args, callbacks=[callback_tune, callback_lightning]) + trainer.fit(model) + + + + + +if __name__== '__main__': + # Execute the hyperparameter search + + config = { + 'num_aug': tune.choice([10, 30]) + } + + # analysis = tune.run( + # train_segmentation, + # config=config, num_samples=1, resources_per_trial={"cpu": 1}, mode='max') + + analysis = tune.run( + tune.with_parameters(train_segmentation, epochs=1, gpus=1), + config=config, num_samples=1, resources_per_trial={"gpu": 1, 'cpu': 4}) \ No newline at end of file From ab9af042d2caf000559173588e56e9aaa9ad1942 Mon Sep 17 00:00:00 2001 From: ihahanov Date: Thu, 9 Dec 2021 16:51:04 +0200 Subject: [PATCH 33/67] add requirements.txt --- requirements.txt | 4 ++++ tuning.py | 7 ++----- 2 files changed, 6 insertions(+), 5 deletions(-) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..9a449a63 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +pytorch-lightning==1.2.9 +ray==1.9.0 +trimesh==3.8.19 +open3d==0.11.2 \ No newline at end of file diff --git a/tuning.py b/tuning.py index 9d838ff7..73b1ca27 100644 --- a/tuning.py +++ b/tuning.py @@ -18,9 +18,6 @@ def train_segmentation(config): trainer.fit(model) - - - if __name__== '__main__': # Execute the hyperparameter search @@ -33,5 +30,5 @@ def train_segmentation(config): # config=config, num_samples=1, resources_per_trial={"cpu": 1}, mode='max') analysis = tune.run( - tune.with_parameters(train_segmentation, epochs=1, gpus=1), - config=config, num_samples=1, resources_per_trial={"gpu": 1, 'cpu': 4}) \ No newline at end of file + tune.with_parameters(train_segmentation), + config=config, num_samples=1, resources_per_trial={"gpu": 1, 'cpu': 1}) \ No newline at end of file From 3b53336ad4d1eeb65e891a4731c98c700dc07249 Mon Sep 17 00:00:00 2001 From: ihahanov Date: Thu, 9 Dec 2021 16:55:52 +0200 Subject: [PATCH 34/67] add tune.sh --- scripts/roof_seg/tune.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 scripts/roof_seg/tune.sh diff --git a/scripts/roof_seg/tune.sh b/scripts/roof_seg/tune.sh new file mode 100644 index 00000000..5e9010d5 --- /dev/null +++ b/scripts/roof_seg/tune.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +## run the training +python tuning.py \ +--dataroot datasets/roof_seg \ +--name roof_seg \ +--arch meshunet \ +--dataset_mode segmentation \ +--ncf 32 64 128 256 \ +--ninput_edges 14000 \ +--pool_res 12000 10500 9000 \ +--resblocks 2 \ +--batch_size 1 \ +--lr 0.01 \ +--num_aug 20 \ +--slide_verts 0.2 +#--from_pretrained checkpoints/coseg_aliens/latest_net.pth \ No newline at end of file From c1fd89051acaa42a48b6fc8f66008ac5862f7d02 Mon Sep 17 00:00:00 2001 From: ihahanov Date: Thu, 9 Dec 2021 16:58:09 +0200 Subject: [PATCH 35/67] make full data path in tune.sh --- scripts/roof_seg/tune.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/roof_seg/tune.sh b/scripts/roof_seg/tune.sh index 5e9010d5..13bb127f 100644 --- a/scripts/roof_seg/tune.sh +++ b/scripts/roof_seg/tune.sh @@ -2,7 +2,7 @@ ## run the training python tuning.py \ ---dataroot datasets/roof_seg \ +--dataroot /home/ihahanov/Projects/roof-measurements/dl_roof_extraction/meshcnn/datasets/roof_seg \ --name roof_seg \ --arch meshunet \ --dataset_mode segmentation \ From 8026d6ef4eb04bcb17f44ec7718a77b003ee3deb Mon Sep 17 00:00:00 2001 From: ihahanov Date: Mon, 13 Dec 2021 11:43:08 +0200 Subject: [PATCH 36/67] add model2-5 to train dataset --- datasets/roof_seg.dvc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datasets/roof_seg.dvc b/datasets/roof_seg.dvc index 5d6e329b..d9c36976 100644 --- a/datasets/roof_seg.dvc +++ b/datasets/roof_seg.dvc @@ -1,5 +1,5 @@ outs: -- md5: 98089000c8f377aeff64977d234c8ffb.dir - size: 163643527 - nfiles: 209 +- md5: 098eceab3385794825f6e35f6c8f76e5.dir + size: 104343136 + nfiles: 210 path: roof_seg From e7d77e9ef17a59d18c883c8aef93c43a812e2222 Mon Sep 17 00:00:00 2001 From: ihahanov Date: Mon, 13 Dec 2021 16:10:33 +0200 Subject: [PATCH 37/67] remove .idea, start adding cloudwatch logs --- train_pl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/train_pl.py b/train_pl.py index 4520b8f1..f657a34e 100644 --- a/train_pl.py +++ b/train_pl.py @@ -87,6 +87,7 @@ def configure_optimizers(self): opt = torch.optim.SGD(self.model.net.parameters(), lr=self.opt.lr, momentum=0.9, weight_decay=0.0002) + # opt = torch.optim.Adam(self.model.net.parameters(), lr=self.opt.lr, weight_decay=0.0002) sched = torch.optim.lr_scheduler.CosineAnnealingLR(opt, self.opt.max_epochs * 2) return [opt], [sched] From a0db6ff82c34b7dbe636e28d640ea7d670fca872 Mon Sep 17 00:00:00 2001 From: ihahanov Date: Mon, 13 Dec 2021 16:28:22 +0200 Subject: [PATCH 38/67] change model checkpoint filename --- train_pl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train_pl.py b/train_pl.py index f657a34e..1954b664 100644 --- a/train_pl.py +++ b/train_pl.py @@ -100,5 +100,5 @@ def configure_optimizers(self): callbacks=[ModelCheckpoint(monitor='val_iou', mode='max', save_top_k=3, - filename='{epoch:02d}-{val_acc_epoch:.2f}',)]) + filename='{epoch:02d}-{val_iou:.2f}',)]) trainer.fit(model) \ No newline at end of file From 54ae53667472b52a0b842e1a832c9e8793a8b221 Mon Sep 17 00:00:00 2001 From: ihahanov Date: Mon, 13 Dec 2021 16:31:01 +0200 Subject: [PATCH 39/67] change default input edges param to 16k --- scripts/roof_seg/train.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/roof_seg/train.sh b/scripts/roof_seg/train.sh index 30989325..9fd822cb 100644 --- a/scripts/roof_seg/train.sh +++ b/scripts/roof_seg/train.sh @@ -7,11 +7,11 @@ python train_pl.py \ --arch meshunet \ --dataset_mode segmentation \ --ncf 32 64 128 256 \ ---ninput_edges 14000 \ +--ninput_edges 16000 \ --pool_res 12000 10500 9000 \ --resblocks 2 \ --batch_size 1 \ ---lr 0.01 \ +--lr 0.005 \ --num_aug 20 \ --slide_verts 0.2 #--from_pretrained checkpoints/coseg_aliens/latest_net.pth \ No newline at end of file From 7a91cf0d64f9bae60c8688eec1f50ca7077ba16f Mon Sep 17 00:00:00 2001 From: ihahanov Date: Mon, 13 Dec 2021 17:14:28 +0200 Subject: [PATCH 40/67] add weights saving into pytorch lightning pipeline --- train_pl.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/train_pl.py b/train_pl.py index 1954b664..10025a63 100644 --- a/train_pl.py +++ b/train_pl.py @@ -24,6 +24,7 @@ def __init__(self, opt): super().__init__() self.opt = opt self.model = create_model(opt) + self.net = self.model.net if opt.from_pretrained is not None: print('Loaded pretrained weights:', opt.from_pretrained) self.model.load_weights(opt.from_pretrained) @@ -87,7 +88,7 @@ def configure_optimizers(self): opt = torch.optim.SGD(self.model.net.parameters(), lr=self.opt.lr, momentum=0.9, weight_decay=0.0002) - # opt = torch.optim.Adam(self.model.net.parameters(), lr=self.opt.lr, weight_decay=0.0002) + # opt = torch.optim.Adam(self.model.net.parameters(), lr.) sched = torch.optim.lr_scheduler.CosineAnnealingLR(opt, self.opt.max_epochs * 2) return [opt], [sched] @@ -100,5 +101,5 @@ def configure_optimizers(self): callbacks=[ModelCheckpoint(monitor='val_iou', mode='max', save_top_k=3, - filename='{epoch:02d}-{val_iou:.2f}',)]) + filename='{epoch:02d}-{val_acc_epoch:.2f}',)]) trainer.fit(model) \ No newline at end of file From 264342fab204eca4c1a669e27bc41221b760ccd9 Mon Sep 17 00:00:00 2001 From: ihahanov Date: Mon, 13 Dec 2021 17:50:55 +0200 Subject: [PATCH 41/67] inherit pl module form classifier model --- train_pl.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/train_pl.py b/train_pl.py index 10025a63..b6748b43 100644 --- a/train_pl.py +++ b/train_pl.py @@ -15,20 +15,19 @@ from models.losses import postprocess from models.losses import ce_jaccard import warnings +from models import networks +from models.mesh_classifier import ClassifierModel warnings.filterwarnings("ignore") -class MeshSegmenter(pl.LightningModule): +class MeshSegmenter(pl.LightningModule, ClassifierModel): def __init__(self, opt): - super().__init__() - self.opt = opt - self.model = create_model(opt) - self.net = self.model.net + ClassifierModel.__init__(self, opt) + pl.LightningModule.__init__(self) if opt.from_pretrained is not None: print('Loaded pretrained weights:', opt.from_pretrained) - self.model.load_weights(opt.from_pretrained) - self.criterion = self.model.criterion + self.load_weights(opt.from_pretrained) if self.training: self.train_metrics = torch.nn.ModuleList([ torchmetrics.Accuracy(num_classes=opt.nclasses, average='macro'), @@ -42,8 +41,7 @@ def __init__(self, opt): ]) def step(self, batch, metrics, metric_prefix=''): - self.model.set_input(batch) - out = self.model.forward() + out = self.forward(batch) true, pred = postprocess(self.model.labels, out) loss = self.criterion(true, pred) @@ -65,8 +63,9 @@ def training_step(self, batch, idx): def validation_step(self, batch, idx): return self.step(batch, self.val_metrics, metric_prefix='val_') - def forward(self, image): - return self.model(image) + def forward(self, batch): + self.set_input(batch) + return ClassifierModel.forward(self) def on_train_epoch_end(self, unused=None): for m in self.train_metrics: @@ -88,7 +87,6 @@ def configure_optimizers(self): opt = torch.optim.SGD(self.model.net.parameters(), lr=self.opt.lr, momentum=0.9, weight_decay=0.0002) - # opt = torch.optim.Adam(self.model.net.parameters(), lr.) sched = torch.optim.lr_scheduler.CosineAnnealingLR(opt, self.opt.max_epochs * 2) return [opt], [sched] @@ -101,5 +99,5 @@ def configure_optimizers(self): callbacks=[ModelCheckpoint(monitor='val_iou', mode='max', save_top_k=3, - filename='{epoch:02d}-{val_acc_epoch:.2f}',)]) + filename='{epoch:02d}-{val_iou:.2f}',)]) trainer.fit(model) \ No newline at end of file From d7d3522d68def635918c80e0abe69fe0bb1bc64f Mon Sep 17 00:00:00 2001 From: ihahanov Date: Mon, 13 Dec 2021 17:59:32 +0200 Subject: [PATCH 42/67] inherit pl module form classifier model --- train_pl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train_pl.py b/train_pl.py index b6748b43..5e00a38f 100644 --- a/train_pl.py +++ b/train_pl.py @@ -23,8 +23,8 @@ class MeshSegmenter(pl.LightningModule, ClassifierModel): def __init__(self, opt): - ClassifierModel.__init__(self, opt) pl.LightningModule.__init__(self) + ClassifierModel.__init__(self, opt) if opt.from_pretrained is not None: print('Loaded pretrained weights:', opt.from_pretrained) self.load_weights(opt.from_pretrained) From c1ab1f207f5dd86e8a595bb6a5cde4fcf991d7d5 Mon Sep 17 00:00:00 2001 From: ihahanov Date: Mon, 13 Dec 2021 22:54:30 +0200 Subject: [PATCH 43/67] fix saving checkpoints for pl, update main with loading pl model --- train_pl.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/train_pl.py b/train_pl.py index 5e00a38f..30b45d1e 100644 --- a/train_pl.py +++ b/train_pl.py @@ -24,7 +24,20 @@ class MeshSegmenter(pl.LightningModule, ClassifierModel): def __init__(self, opt): pl.LightningModule.__init__(self) - ClassifierModel.__init__(self, opt) + self.opt = opt + self.gpu_ids = opt.gpu_ids + self.optimizer = None + self.edge_features = None + self.labels = None + self.mesh = None + self.soft_label = None + self.loss = None + self.nclasses = opt.nclasses + + # load/define networks + self.net = networks.define_classifier(opt.input_nc, opt.ncf, opt.ninput_edges, opt.nclasses, opt, + self.gpu_ids, opt.arch, opt.init_type, opt.init_gain) + self.criterion = networks.define_loss(opt) if opt.from_pretrained is not None: print('Loaded pretrained weights:', opt.from_pretrained) self.load_weights(opt.from_pretrained) @@ -42,7 +55,7 @@ def __init__(self, opt): def step(self, batch, metrics, metric_prefix=''): out = self.forward(batch) - true, pred = postprocess(self.model.labels, out) + true, pred = postprocess(self.labels, out) loss = self.criterion(true, pred) true = true.view(-1) @@ -63,9 +76,11 @@ def training_step(self, batch, idx): def validation_step(self, batch, idx): return self.step(batch, self.val_metrics, metric_prefix='val_') - def forward(self, batch): - self.set_input(batch) - return ClassifierModel.forward(self) + def forward(self, data): + input_edge_features = torch.from_numpy(data['edge_features']).float() + self.edge_features = input_edge_features.to(self.device).requires_grad_(self.training) + self.mesh = data['mesh'] + return self.net(self.edge_features, self.mesh) def on_train_epoch_end(self, unused=None): for m in self.train_metrics: @@ -84,7 +99,7 @@ def val_dataloader(self): return DataLoader(self.opt) def configure_optimizers(self): - opt = torch.optim.SGD(self.model.net.parameters(), lr=self.opt.lr, + opt = torch.optim.SGD(self.net.parameters(), lr=self.opt.lr, momentum=0.9, weight_decay=0.0002) sched = torch.optim.lr_scheduler.CosineAnnealingLR(opt, self.opt.max_epochs * 2) From c4d63c7bfd1fff6f9c7a279cca734f3eb86ac7dc Mon Sep 17 00:00:00 2001 From: ihahanov Date: Mon, 13 Dec 2021 23:02:59 +0200 Subject: [PATCH 44/67] revert labels storing in model during training --- train_pl.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/train_pl.py b/train_pl.py index 30b45d1e..2b912a06 100644 --- a/train_pl.py +++ b/train_pl.py @@ -78,6 +78,8 @@ def validation_step(self, batch, idx): def forward(self, data): input_edge_features = torch.from_numpy(data['edge_features']).float() + if 'label' in data: + self.labels = torch.from_numpy(data['label']).long().to(self.device) self.edge_features = input_edge_features.to(self.device).requires_grad_(self.training) self.mesh = data['mesh'] return self.net(self.edge_features, self.mesh) From f3c474855da4918a4c35f5ba4f7dbbdd4ffa84bd Mon Sep 17 00:00:00 2001 From: ihahanov Date: Tue, 14 Dec 2021 10:20:34 +0200 Subject: [PATCH 45/67] add warmuo --- options/pl_options.py | 1 + train_pl.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/options/pl_options.py b/options/pl_options.py index 14406472..4cddc924 100644 --- a/options/pl_options.py +++ b/options/pl_options.py @@ -5,6 +5,7 @@ def initialize(self): TrainOptions.initialize(self) self.parser.add_argument('--gpus', type=int, default=1) self.parser.add_argument('--max_epochs', type=int, default=200) + self.parser.add_argument('--warmup_epochs', type=int, default=50) self.parser.add_argument('--nclasses', type=int, default=2) self.parser.add_argument('--input_nc', type=int, default=5) self.parser.add_argument('--class_weights', nargs='+', default=[0.5, 2], type=float) diff --git a/train_pl.py b/train_pl.py index 2b912a06..20c74a79 100644 --- a/train_pl.py +++ b/train_pl.py @@ -104,7 +104,8 @@ def configure_optimizers(self): opt = torch.optim.SGD(self.net.parameters(), lr=self.opt.lr, momentum=0.9, weight_decay=0.0002) - sched = torch.optim.lr_scheduler.CosineAnnealingLR(opt, self.opt.max_epochs * 2) + sched = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(opt, self.opt.warmup_epochs) + # sched = torch.optim.lr_scheduler.CosineAnnealingLR(opt, self.opt.max_epochs * 2) return [opt], [sched] From 7bd86a04d51ca1fd5b3a3f16954b09cfda7a61e7 Mon Sep 17 00:00:00 2001 From: ihahanov Date: Tue, 14 Dec 2021 12:48:21 +0200 Subject: [PATCH 46/67] add model 6-8 to dataset --- data/make_annotation_from_vertex_labels.py | 1 + datasets/roof_seg.dvc | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/data/make_annotation_from_vertex_labels.py b/data/make_annotation_from_vertex_labels.py index a1bdae7c..9322e154 100644 --- a/data/make_annotation_from_vertex_labels.py +++ b/data/make_annotation_from_vertex_labels.py @@ -162,6 +162,7 @@ def show_mesh(edges, vs, label, colors=[[0,0,0,255], [120,120,120,255]]): def create_files(path): for filename in glob.glob(os.path.join(path, 'obj/*.obj')): + print(filename) basename = os.path.splitext(os.path.basename(filename))[0] v_label_name = os.path.join(os.path.join(path, 'vseg'), basename + '.eseg') label_name = os.path.join(os.path.join(path, 'seg'), basename + '.eseg') diff --git a/datasets/roof_seg.dvc b/datasets/roof_seg.dvc index d9c36976..da0e143a 100644 --- a/datasets/roof_seg.dvc +++ b/datasets/roof_seg.dvc @@ -1,5 +1,5 @@ outs: -- md5: 098eceab3385794825f6e35f6c8f76e5.dir - size: 104343136 - nfiles: 210 +- md5: ce9705f115060277b96c148939f6d8ba.dir + size: 124646029 + nfiles: 235 path: roof_seg From f57fc9c16d3d18c3a9514b06bf4d610ee221c9b1 Mon Sep 17 00:00:00 2001 From: ihahanov Date: Tue, 14 Dec 2021 13:01:50 +0200 Subject: [PATCH 47/67] fix test dataset --- datasets/roof_seg.dvc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datasets/roof_seg.dvc b/datasets/roof_seg.dvc index da0e143a..82d65d73 100644 --- a/datasets/roof_seg.dvc +++ b/datasets/roof_seg.dvc @@ -1,5 +1,5 @@ outs: -- md5: ce9705f115060277b96c148939f6d8ba.dir - size: 124646029 +- md5: 92617404486d7ffdf64e1fad503913b8.dir + size: 126349624 nfiles: 235 path: roof_seg From ea7c6f9138a805d57252cce67e55c40ddf69467f Mon Sep 17 00:00:00 2001 From: ihahanov Date: Tue, 14 Dec 2021 13:05:02 +0200 Subject: [PATCH 48/67] fix train dataset --- datasets/roof_seg.dvc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datasets/roof_seg.dvc b/datasets/roof_seg.dvc index 82d65d73..c4ee6f32 100644 --- a/datasets/roof_seg.dvc +++ b/datasets/roof_seg.dvc @@ -1,5 +1,5 @@ outs: -- md5: 92617404486d7ffdf64e1fad503913b8.dir - size: 126349624 +- md5: d34371729ce4625b518a8f5a39b38046.dir + size: 127136382 nfiles: 235 path: roof_seg From f34ce83536cb6172b155ab3f4b9cbac4ef2f6094 Mon Sep 17 00:00:00 2001 From: ihahanov Date: Tue, 14 Dec 2021 13:23:43 +0200 Subject: [PATCH 49/67] fix model 6-8 --- datasets/roof_seg.dvc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datasets/roof_seg.dvc b/datasets/roof_seg.dvc index c4ee6f32..a446f4f2 100644 --- a/datasets/roof_seg.dvc +++ b/datasets/roof_seg.dvc @@ -1,5 +1,5 @@ outs: -- md5: d34371729ce4625b518a8f5a39b38046.dir - size: 127136382 +- md5: 8ecc5506cc0810336049247d44738dae.dir + size: 127052178 nfiles: 235 path: roof_seg From cb9fc57c0dbc8873601491d74b0e36d6d2e71a4f Mon Sep 17 00:00:00 2001 From: ihahanov Date: Tue, 14 Dec 2021 15:35:06 +0200 Subject: [PATCH 50/67] add model 9-11 to dataset --- datasets/roof_seg.dvc | 6 +++--- scripts/roof_seg/train.sh | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/datasets/roof_seg.dvc b/datasets/roof_seg.dvc index a446f4f2..3108f073 100644 --- a/datasets/roof_seg.dvc +++ b/datasets/roof_seg.dvc @@ -1,5 +1,5 @@ outs: -- md5: 8ecc5506cc0810336049247d44738dae.dir - size: 127052178 - nfiles: 235 +- md5: 49739fa280878f2912e32a10b4dd9d72.dir + size: 141819574 + nfiles: 266 path: roof_seg diff --git a/scripts/roof_seg/train.sh b/scripts/roof_seg/train.sh index 9fd822cb..78aa7f5a 100644 --- a/scripts/roof_seg/train.sh +++ b/scripts/roof_seg/train.sh @@ -9,9 +9,11 @@ python train_pl.py \ --ncf 32 64 128 256 \ --ninput_edges 16000 \ --pool_res 12000 10500 9000 \ ---resblocks 2 \ +--resblocks 3 \ --batch_size 1 \ --lr 0.005 \ --num_aug 20 \ ---slide_verts 0.2 +--slide_verts 0.2 \ +--warmup_epochs 300 + #--from_pretrained checkpoints/coseg_aliens/latest_net.pth \ No newline at end of file From dc46c9d5b03fd0d65a2936820f4f396143b313bc Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 16 Dec 2021 18:10:48 +0000 Subject: [PATCH 51/67] config update and results saving --- train_pl.py | 4 ++-- tuning.py | 40 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/train_pl.py b/train_pl.py index 4520b8f1..3828f8eb 100644 --- a/train_pl.py +++ b/train_pl.py @@ -30,12 +30,12 @@ def __init__(self, opt): self.criterion = self.model.criterion if self.training: self.train_metrics = torch.nn.ModuleList([ - torchmetrics.Accuracy(num_classes=opt.nclasses, average='macro'), + torchmetrics.Accuracy(),# (num_classes=opt.nclasses, average='macro'), torchmetrics.IoU(num_classes=opt.nclasses), torchmetrics.F1(num_classes=opt.nclasses, average='macro') ]) self.val_metrics = torch.nn.ModuleList([ - torchmetrics.Accuracy(num_classes=opt.nclasses, average='macro'), + torchmetrics.Accuracy(), #num_classes=opt.nclasses, average='macro'), torchmetrics.IoU(num_classes=opt.nclasses), torchmetrics.F1(num_classes=opt.nclasses, average='macro') ]) diff --git a/tuning.py b/tuning.py index 73b1ca27..ce3be025 100644 --- a/tuning.py +++ b/tuning.py @@ -1,3 +1,5 @@ +import json + from pytorch_lightning.callbacks import ModelCheckpoint from pytorch_lightning import Trainer from train_pl import * @@ -9,11 +11,26 @@ def train_segmentation(config): # args = PLOptions().parse() - args.num_aug = config.get('num_aug') + if config.get('num_aug'): + args.num_aug = config.get('num_aug') + if config.get('resblocks'): + args.resblocks = config.get('resblocks') + if config.get('ncf'): + args.ncf = config.get('ncf') + if config.get('slide_verts'): + args.slide_verts = config.get('slide_verts') + if config.get('lr'): + args.lr = config.get('lr') + model = MeshSegmenter(args) callback_tune = TuneReportCallback(metrics='val_iou', on="validation_end") callback_lightning = ModelCheckpoint(monitor='val_iou', mode='max', save_top_k=3, filename='{epoch:02d}-{val_acc_epoch:.2f}', ) + + # callback_tune_f1 = TuneReportCallback(metrics='val_f1', on="validation_end") + # callback_lightning_f1 = ModelCheckpoint(monitor='val_f1', mode='max', save_top_k=3, + # filename='{epoch:02d}-{val_acc_epoch:.2f}', ) + trainer = Trainer.from_argparse_args(args, callbacks=[callback_tune, callback_lightning]) trainer.fit(model) @@ -22,13 +39,30 @@ def train_segmentation(config): # Execute the hyperparameter search config = { - 'num_aug': tune.choice([10, 30]) + # 'num_aug': tune.grid_search([10, 20, 30]), + # 'resblocks': tune.grid_search([2, 3, 4, 5]), + 'ncf': tune.grid_search([[64, 128, 256, 512], [32, 64, 128, 256]]), + # 'slide_verts': tune.grid_search([0.08, 0.1, 0.12, 0.16, 0.2]), + # 'lr': tune.grid_search([0.00005, 0.0002, 0.0005]) } + ## CPU only # analysis = tune.run( # train_segmentation, # config=config, num_samples=1, resources_per_trial={"cpu": 1}, mode='max') + # GPU analysis = tune.run( tune.with_parameters(train_segmentation), - config=config, num_samples=1, resources_per_trial={"gpu": 1, 'cpu': 1}) \ No newline at end of file + config=config, num_samples=1, resources_per_trial={"gpu": 1, 'cpu': 1}) + + # Saving the results + best_config = analysis.get_best_config(metric='val_iou', mode="max") + print("Best config: ", best_config) + + file = open(os.path.join(args.checkpoints_dir, 'roof_seg', 'best_config.json'), 'w') + json.dump(best_config, file) + + # Get a dataframe for analyzing trial results. + df = analysis.results_df + print(df) \ No newline at end of file From d040d9b0bccf9ddd6ff09245e4d0be83ea1f3346 Mon Sep 17 00:00:00 2001 From: ihahanov Date: Fri, 17 Dec 2021 15:19:57 +0200 Subject: [PATCH 52/67] add model 11-14 --- datasets/roof_seg.dvc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datasets/roof_seg.dvc b/datasets/roof_seg.dvc index 3108f073..58fc18f2 100644 --- a/datasets/roof_seg.dvc +++ b/datasets/roof_seg.dvc @@ -1,5 +1,5 @@ outs: -- md5: 49739fa280878f2912e32a10b4dd9d72.dir - size: 141819574 - nfiles: 266 +- md5: 07de8ae87eb0955a8fc3541667a7be73.dir + size: 150377339 + nfiles: 285 path: roof_seg From e28ad487a267b10773a7fa5a26ecd14aa7bc80cc Mon Sep 17 00:00:00 2001 From: ihahanov Date: Fri, 17 Dec 2021 15:45:00 +0200 Subject: [PATCH 53/67] change sgd to adam --- train_pl.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/train_pl.py b/train_pl.py index b346ba89..0aecc49c 100644 --- a/train_pl.py +++ b/train_pl.py @@ -101,9 +101,10 @@ def val_dataloader(self): return DataLoader(self.opt) def configure_optimizers(self): - opt = torch.optim.SGD(self.net.parameters(), lr=self.opt.lr, - momentum=0.9, - weight_decay=0.0002) + opt = torch.optim.Adam(self.net.parameters(), lr=self.opt.lr, weight_decay=0.0002) + # opt = torch.optim.SGD(self.net.parameters(), lr=self.opt.lr, + # momentum=0.9, + # weight_decay=0.0002) sched = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(opt, self.opt.warmup_epochs) # sched = torch.optim.lr_scheduler.CosineAnnealingLR(opt, self.opt.max_epochs * 2) return [opt], [sched] From 2a2fd7817963933200c3c9e94d9ef761b630e3fe Mon Sep 17 00:00:00 2001 From: ihahanov Date: Mon, 20 Dec 2021 12:00:15 +0200 Subject: [PATCH 54/67] add cache cleaning after each run --- tuning.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tuning.py b/tuning.py index ce3be025..f46dec66 100644 --- a/tuning.py +++ b/tuning.py @@ -1,5 +1,6 @@ import json +import torch.cuda from pytorch_lightning.callbacks import ModelCheckpoint from pytorch_lightning import Trainer from train_pl import * @@ -33,6 +34,7 @@ def train_segmentation(config): trainer = Trainer.from_argparse_args(args, callbacks=[callback_tune, callback_lightning]) trainer.fit(model) + torch.cuda.empty_cache() if __name__== '__main__': From 0b90e25c9582c8f516f2f705fc43abd47b957406 Mon Sep 17 00:00:00 2001 From: ihahanov Date: Mon, 20 Dec 2021 12:03:34 +0200 Subject: [PATCH 55/67] add warmup epochs to tune.sh --- scripts/roof_seg/tune.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/roof_seg/tune.sh b/scripts/roof_seg/tune.sh index 13bb127f..d4db7ee7 100644 --- a/scripts/roof_seg/tune.sh +++ b/scripts/roof_seg/tune.sh @@ -13,5 +13,6 @@ python tuning.py \ --batch_size 1 \ --lr 0.01 \ --num_aug 20 \ ---slide_verts 0.2 +--slide_verts 0.2 \ +--warmup_epochs 300 #--from_pretrained checkpoints/coseg_aliens/latest_net.pth \ No newline at end of file From 380fb332a89625f818c375a3dde61ef36ada74c0 Mon Sep 17 00:00:00 2001 From: ihahanov Date: Mon, 20 Dec 2021 13:24:15 +0200 Subject: [PATCH 56/67] fix abs path in tune.sh. increase ninput edges --- scripts/roof_seg/tune.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/roof_seg/tune.sh b/scripts/roof_seg/tune.sh index d4db7ee7..50831185 100644 --- a/scripts/roof_seg/tune.sh +++ b/scripts/roof_seg/tune.sh @@ -2,12 +2,12 @@ ## run the training python tuning.py \ ---dataroot /home/ihahanov/Projects/roof-measurements/dl_roof_extraction/meshcnn/datasets/roof_seg \ +--dataroot $(pwd)/datasets/roof_seg \ --name roof_seg \ --arch meshunet \ --dataset_mode segmentation \ --ncf 32 64 128 256 \ ---ninput_edges 14000 \ +--ninput_edges 16000 \ --pool_res 12000 10500 9000 \ --resblocks 2 \ --batch_size 1 \ From 15ec9e1efdf505ba76890bc44e14a179f859c4d2 Mon Sep 17 00:00:00 2001 From: ihahanov Date: Mon, 20 Dec 2021 13:25:13 +0200 Subject: [PATCH 57/67] reduce epochs to 50 in tune.sh --- scripts/roof_seg/tune.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/roof_seg/tune.sh b/scripts/roof_seg/tune.sh index 50831185..751efcfa 100644 --- a/scripts/roof_seg/tune.sh +++ b/scripts/roof_seg/tune.sh @@ -14,5 +14,6 @@ python tuning.py \ --lr 0.01 \ --num_aug 20 \ --slide_verts 0.2 \ ---warmup_epochs 300 +--warmup_epochs 70 \ +--max_epochs 50 #--from_pretrained checkpoints/coseg_aliens/latest_net.pth \ No newline at end of file From a836557882285575e54e9cb66160d7d1ed7cd600 Mon Sep 17 00:00:00 2001 From: ihahanov Date: Mon, 20 Dec 2021 15:07:35 +0200 Subject: [PATCH 58/67] update tuning.py, add more parameters to tune --- options/pl_options.py | 2 ++ train_pl.py | 12 ++++++++---- tuning.py | 27 ++++++++++----------------- 3 files changed, 20 insertions(+), 21 deletions(-) diff --git a/options/pl_options.py b/options/pl_options.py index 4cddc924..032bdb18 100644 --- a/options/pl_options.py +++ b/options/pl_options.py @@ -10,6 +10,8 @@ def initialize(self): self.parser.add_argument('--input_nc', type=int, default=5) self.parser.add_argument('--class_weights', nargs='+', default=[0.5, 2], type=float) self.parser.add_argument('--from_pretrained', type=str, default=None) + self.parser.add_argument('--optimizer', choices=['adam', 'sgd', 'adamw'], type=str, default='adam') + self.parser.add_argument('--weight_decay', type=float, default=0.0002) self.parser.add_argument('--progress_bar_refresh_rate', type=int, default=20) self.parser.add_argument('--default_root_dir', default='checkpoints/', diff --git a/train_pl.py b/train_pl.py index 0aecc49c..cbe6db9b 100644 --- a/train_pl.py +++ b/train_pl.py @@ -101,10 +101,14 @@ def val_dataloader(self): return DataLoader(self.opt) def configure_optimizers(self): - opt = torch.optim.Adam(self.net.parameters(), lr=self.opt.lr, weight_decay=0.0002) - # opt = torch.optim.SGD(self.net.parameters(), lr=self.opt.lr, - # momentum=0.9, - # weight_decay=0.0002) + if self.opt.optimizer == 'adam': + opt = torch.optim.Adam(self.net.parameters(), lr=self.opt.lr, weight_decay=self.opt.weight_decay) + elif self.opt.optimizer == 'sgd': + opt = torch.optim.SGD(self.net.parameters(), lr=self.opt.lr, + momentum=0.9, + weight_decay=self.opt.weight_decay) + elif self.opt.optimizer == 'adamw': + opt = torch.optim.AdamW(self.net.parameters(), lr=self.opt.lf, weight_decay=self.opt.weight_decay) sched = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(opt, self.opt.warmup_epochs) # sched = torch.optim.lr_scheduler.CosineAnnealingLR(opt, self.opt.max_epochs * 2) return [opt], [sched] diff --git a/tuning.py b/tuning.py index f46dec66..80c7d2bf 100644 --- a/tuning.py +++ b/tuning.py @@ -8,20 +8,11 @@ from ray.tune.integration.pytorch_lightning import TuneReportCallback -args = PLOptions().parse() def train_segmentation(config): - # args = PLOptions().parse() - if config.get('num_aug'): - args.num_aug = config.get('num_aug') - if config.get('resblocks'): - args.resblocks = config.get('resblocks') - if config.get('ncf'): - args.ncf = config.get('ncf') - if config.get('slide_verts'): - args.slide_verts = config.get('slide_verts') - if config.get('lr'): - args.lr = config.get('lr') + args = PLOptions().parse() + for k, v in config.items(): + args.__dict__[k] = v model = MeshSegmenter(args) callback_tune = TuneReportCallback(metrics='val_iou', on="validation_end") @@ -41,11 +32,13 @@ def train_segmentation(config): # Execute the hyperparameter search config = { - # 'num_aug': tune.grid_search([10, 20, 30]), - # 'resblocks': tune.grid_search([2, 3, 4, 5]), - 'ncf': tune.grid_search([[64, 128, 256, 512], [32, 64, 128, 256]]), - # 'slide_verts': tune.grid_search([0.08, 0.1, 0.12, 0.16, 0.2]), - # 'lr': tune.grid_search([0.00005, 0.0002, 0.0005]) + 'resblocks': tune.grid_search([2, 3, 4]), + 'ncf': tune.grid_search([[64, 128, 256, 512], [32, 64, 128, 256], [16, 32, 64, 128]]), + 'slide_verts': tune.grid_search([0.1, 0.2]), + 'lr': tune.grid_search([0.01, 0.001]), + 'optimizer': tune.grid_search(['adam', 'sgd', 'adamw']), + 'warmup_epochs': tune.grid_search([200, 100, 50]), + 'weight_decay': tune.grid_search([0, 0.0002]), } ## CPU only From 027051d4b8f74eddbaeba5f10f2a3c26d2139d5a Mon Sep 17 00:00:00 2001 From: ihahanov Date: Mon, 20 Dec 2021 15:08:02 +0200 Subject: [PATCH 59/67] set tuning epochs to 100 --- scripts/roof_seg/tune.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/roof_seg/tune.sh b/scripts/roof_seg/tune.sh index 751efcfa..ab49d885 100644 --- a/scripts/roof_seg/tune.sh +++ b/scripts/roof_seg/tune.sh @@ -15,5 +15,5 @@ python tuning.py \ --num_aug 20 \ --slide_verts 0.2 \ --warmup_epochs 70 \ ---max_epochs 50 +--max_epochs 100 #--from_pretrained checkpoints/coseg_aliens/latest_net.pth \ No newline at end of file From e3ec92af09d7f69741a745f3cf9096b98771347c Mon Sep 17 00:00:00 2001 From: ihahanov Date: Mon, 20 Dec 2021 15:11:47 +0200 Subject: [PATCH 60/67] remove redundant params --- scripts/roof_seg/tune.sh | 5 ----- tuning.py | 4 +--- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/scripts/roof_seg/tune.sh b/scripts/roof_seg/tune.sh index ab49d885..1c9b3521 100644 --- a/scripts/roof_seg/tune.sh +++ b/scripts/roof_seg/tune.sh @@ -6,14 +6,9 @@ python tuning.py \ --name roof_seg \ --arch meshunet \ --dataset_mode segmentation \ ---ncf 32 64 128 256 \ --ninput_edges 16000 \ --pool_res 12000 10500 9000 \ ---resblocks 2 \ --batch_size 1 \ ---lr 0.01 \ --num_aug 20 \ ---slide_verts 0.2 \ ---warmup_epochs 70 \ --max_epochs 100 #--from_pretrained checkpoints/coseg_aliens/latest_net.pth \ No newline at end of file diff --git a/tuning.py b/tuning.py index 80c7d2bf..8c2d5e46 100644 --- a/tuning.py +++ b/tuning.py @@ -7,10 +7,9 @@ from ray import tune from ray.tune.integration.pytorch_lightning import TuneReportCallback - +args = PLOptions().parse() def train_segmentation(config): - args = PLOptions().parse() for k, v in config.items(): args.__dict__[k] = v @@ -30,7 +29,6 @@ def train_segmentation(config): if __name__== '__main__': # Execute the hyperparameter search - config = { 'resblocks': tune.grid_search([2, 3, 4]), 'ncf': tune.grid_search([[64, 128, 256, 512], [32, 64, 128, 256], [16, 32, 64, 128]]), From 4e3336bbf415a821c54e4ad24be90d4a1b722895 Mon Sep 17 00:00:00 2001 From: ihahanov Date: Wed, 22 Dec 2021 13:36:23 +0200 Subject: [PATCH 61/67] add roof extraction based on network prediction. add filtration by positive prediction percentage --- tuning.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tuning.py b/tuning.py index 8c2d5e46..3a0cf54a 100644 --- a/tuning.py +++ b/tuning.py @@ -16,7 +16,7 @@ def train_segmentation(config): model = MeshSegmenter(args) callback_tune = TuneReportCallback(metrics='val_iou', on="validation_end") callback_lightning = ModelCheckpoint(monitor='val_iou', mode='max', save_top_k=3, - filename='{epoch:02d}-{val_acc_epoch:.2f}', ) + filename='{epoch:02d}-{val_iou:.2f}', ) # callback_tune_f1 = TuneReportCallback(metrics='val_f1', on="validation_end") # callback_lightning_f1 = ModelCheckpoint(monitor='val_f1', mode='max', save_top_k=3, From 85015fd53b576dcb317896d949121b0a32b598f3 Mon Sep 17 00:00:00 2001 From: ihahanov Date: Wed, 29 Dec 2021 16:17:22 +0200 Subject: [PATCH 62/67] fix tests --- data/blender_scripts/extract_vertex_labels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/blender_scripts/extract_vertex_labels.py b/data/blender_scripts/extract_vertex_labels.py index fffef089..f9fd2ba6 100644 --- a/data/blender_scripts/extract_vertex_labels.py +++ b/data/blender_scripts/extract_vertex_labels.py @@ -12,5 +12,5 @@ label.append(str(0)) with open('/home/ihahanov/Projects/roof-measurements/dl_roof_extraction/meshcnn' - '/datasets/roof_seg/vseg/2510 Garrison Simplified + Annotated.eseg', 'w') as f: + '/datasets/roof_seg/vseg/model12.eseg', 'w') as f: f.write('\n'.join(label)) From 9b9a63deaddaf35d9dac1ef9334424eaa930ed41 Mon Sep 17 00:00:00 2001 From: ihahanov Date: Tue, 4 Jan 2022 13:19:40 +0200 Subject: [PATCH 63/67] fix loading network on cpu --- models/networks.py | 2 +- options/base_options.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/models/networks.py b/models/networks.py index 816baa46..2d9e9ec2 100644 --- a/models/networks.py +++ b/models/networks.py @@ -88,7 +88,7 @@ def init_net(net, init_type, init_gain, gpu_ids): assert(torch.cuda.is_available()) net.cuda(gpu_ids[0]) net = net.cuda() - net = torch.nn.DataParallel(net, gpu_ids) + net = torch.nn.DataParallel(net, gpu_ids) if init_type != 'none': init_weights(net, init_type, init_gain) return net diff --git a/options/base_options.py b/options/base_options.py index 1364a6b1..1d053a65 100644 --- a/options/base_options.py +++ b/options/base_options.py @@ -43,10 +43,10 @@ def initialize(self): # self.initialized = True - def parse(self): + def parse(self, args=None): if not self.initialized: self.initialize() - self.opt, unknown = self.parser.parse_known_args() + self.opt, unknown = self.parser.parse_known_args(args) self.opt.is_train = self.is_train # train or test str_ids = self.opt.gpu_ids.split(',') From ca580674079372f9fb0c73eb4f998b0237c0894f Mon Sep 17 00:00:00 2001 From: ihahanov Date: Tue, 4 Jan 2022 18:31:45 +0200 Subject: [PATCH 64/67] fix loading model without gpu --- models/networks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/networks.py b/models/networks.py index 2d9e9ec2..8e4634e9 100644 --- a/models/networks.py +++ b/models/networks.py @@ -88,7 +88,7 @@ def init_net(net, init_type, init_gain, gpu_ids): assert(torch.cuda.is_available()) net.cuda(gpu_ids[0]) net = net.cuda() - net = torch.nn.DataParallel(net, gpu_ids) + net = torch.nn.DataParallel(net, gpu_ids if gpu_ids else [0]) if init_type != 'none': init_weights(net, init_type, init_gain) return net From db7af9c8fdf8f49816ad3c7a79c220f2c809f295 Mon Sep 17 00:00:00 2001 From: mrybakova Date: Thu, 6 Jan 2022 22:37:12 +0200 Subject: [PATCH 65/67] add decimation --- decimate.py | 108 ++++++++++++++++++++++++++++++++++++++++++ options/pl_options.py | 6 +++ 2 files changed, 114 insertions(+) create mode 100644 decimate.py diff --git a/decimate.py b/decimate.py new file mode 100644 index 00000000..e8dad0e7 --- /dev/null +++ b/decimate.py @@ -0,0 +1,108 @@ +from copy import deepcopy +import os +from collections import OrderedDict + +import trimesh as tm +import numpy as np +import torch + +from options.pl_options import PLOptions +from data import DataLoader +from data.segmentation_data import Mesh +from util.util import pad +from train_pl import MeshSegmenter + + +def show_mesh(mesh, label): + edges = mesh.edges + vertices = mesh.vs + vertex_label = np.zeros(len(vertices)) + for e_l, e in zip(label[0], edges): + if e_l == 1: + vertex_label[e] = 1 + faces = mesh.faces + vertex_colors = np.array([[255, 100, 0, 255], [0, 100, 255, 255]])[vertex_label.astype(int)] + trimesh = tm.Trimesh(faces=faces, vertices=vertices, vertex_colors=vertex_colors) + trimesh.show() + return trimesh + + +def simplify_rooftop(roof_segment: tm.Trimesh, n_triangles) -> tm.Trimesh: + """ + Perform mesh simplificaiton based on desired triangles number + :param roof_segment: Trimesh - submesh of the roof + :param n_triangles: int - number of triangles the simplified mesh would contain + :return: tm.Trimesh - Simplified mesh + """ + n_triangles = max([n_triangles, 5]) + segment = roof_segment.simplify_quadratic_decimation(n_triangles) + + return segment + + +def load_obj(path, opt, mean, std): + mesh = Mesh(file=path, opt=opt, hold_history=True, export_folder=opt.export_folder) + meta = {} + meta['mesh'] = [mesh] + meta['path'] = [path] + edge_features = mesh.extract_features() + edge_features = pad(edge_features, opt.ninput_edges) + edge_features = (edge_features - mean) / std + meta['edge_features'] = np.expand_dims(edge_features, 0) + meta['label'] = np.array([]) + meta['soft_label'] = np.array([]) + return meta + + +def run_decimation(epoch=-1): + opt = PLOptions().parse() + opt.serial_batches = True # no shuffle + dataset = DataLoader(opt) + model = MeshSegmenter(opt) + + device = torch.device('cuda:{}'.format(opt.gpu_ids[0])) if opt.gpu_ids else torch.device('cpu') + checkpoint = torch.load(opt.model_path, map_location=device) + + state_dict = checkpoint['state_dict'] + new_state_dict = OrderedDict() + for k, v in state_dict.items(): + new_key = k.replace('.module', '') + new_state_dict[new_key] = v + model.load_state_dict(new_state_dict) + + for i, data in enumerate(dataset): + if i != 21 and i != 22: + continue + print(i, data['path']) + obj_name = os.path.basename(data['path'][0]) + print(obj_name) + + torch.cuda.empty_cache() + + mesh = deepcopy(data['mesh'][0]) + pred_class = model.forward(data).max(1)[1] + tm_mesh = show_mesh(mesh, label=pred_class) + + torch.cuda.empty_cache() + + for desired_triangle_area in [0.3, 8]:#[0.5, 0.8, 1, 1.2, 1.4, 1.7, 2, 2.5, 3.5, 5, 7]: + print(desired_triangle_area) + + tm_mesh_new = simplify_rooftop(tm_mesh, int((tm_mesh.area / desired_triangle_area))) + + new_obj_name = obj_name[:-4] + '_' + str(desired_triangle_area) + '.obj' + obj_path = os.path.join(opt.decimation_dir, new_obj_name) + + with open(obj_path, mode='w') as f: + f.write(tm.exchange.obj.export_obj(tm_mesh_new)) + data_new = load_obj(obj_path, opt, dataset.dataset.mean, dataset.dataset.std) + + mesh_new = deepcopy(data_new['mesh'][0]) + pred_class_new = model.forward(data_new).max(1)[1] + show_mesh(mesh_new, label=pred_class_new) + + torch.cuda.empty_cache() + # os.unlink(f.name) + +if __name__ == '__main__': + run_decimation() diff --git a/options/pl_options.py b/options/pl_options.py index 032bdb18..e2cd7949 100644 --- a/options/pl_options.py +++ b/options/pl_options.py @@ -16,3 +16,9 @@ def initialize(self): self.parser.add_argument('--progress_bar_refresh_rate', type=int, default=20) self.parser.add_argument('--default_root_dir', default='checkpoints/', help='pytorch-lightning log path') + # options used for decimation script only + self.parser.add_argument('--model_path', default='checkpoints/lightning_logs/version_0/checkpoints/epoch=95-val_acc_epoch=0.00.ckpt', + help='.ckpt file with trained model') + self.parser.add_argument('--decimation_dir', + default='datasets/roof_seg/obj_new', + help='augmented meshes are saved here') From 5b1e58f5e09cca1262d5c8b602e4810b698daaac Mon Sep 17 00:00:00 2001 From: mrybakova Date: Thu, 6 Jan 2022 22:38:50 +0200 Subject: [PATCH 66/67] add decimation --- decimate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/decimate.py b/decimate.py index e8dad0e7..c359cb98 100644 --- a/decimate.py +++ b/decimate.py @@ -85,7 +85,7 @@ def run_decimation(epoch=-1): torch.cuda.empty_cache() - for desired_triangle_area in [0.3, 8]:#[0.5, 0.8, 1, 1.2, 1.4, 1.7, 2, 2.5, 3.5, 5, 7]: + for desired_triangle_area in [0.5, 0.8, 1, 1.2, 1.4, 1.7, 2, 2.5, 3.5, 5, 7]: print(desired_triangle_area) tm_mesh_new = simplify_rooftop(tm_mesh, int((tm_mesh.area / desired_triangle_area))) From d99b7fc62b632151a7e4c51c9f65c53dbbc91469 Mon Sep 17 00:00:00 2001 From: mrybakova Date: Thu, 6 Jan 2022 20:59:17 +0000 Subject: [PATCH 67/67] add decimated models to dataset --- datasets/roof_seg.dvc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datasets/roof_seg.dvc b/datasets/roof_seg.dvc index 58fc18f2..fe5014e9 100644 --- a/datasets/roof_seg.dvc +++ b/datasets/roof_seg.dvc @@ -1,5 +1,5 @@ outs: -- md5: 07de8ae87eb0955a8fc3541667a7be73.dir - size: 150377339 - nfiles: 285 +- md5: 5ad27ad6392a64493c0976a5ad1a8296.dir + size: 176735174 + nfiles: 404 path: roof_seg