fairness_ssl/tune_hparam.py at main · googleinterns/fairness_ssl · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
"""Tuning Hyper-parameters from a given directory"""

from absl import app
from absl import flags
from numpy import genfromtxt
import numpy as np

from util.utils import HParams

import os
import pdb
import csv

flags.DEFINE_string(name='directory', default='results_all/Adult2_GroupDRO/', help='The directory to traverse')
flags.DEFINE_list(name='seed_list', default=['42', '43', '44'],
                  help='Marginal probabilities for each group.')
flags.DEFINE_integer(name='min_c', default=2, help='index of the minority group')
flags.DEFINE_integer(name='nvp_th', default=5, help='chooses best worstoff accuracy from top nvp_th accuracies')
flags.DEFINE_integer(name='nvp_percent', default=98, help='chooses best worstoff accuracy from top nvp_th accuracies')

FLAGS = flags.FLAGS

def main(unused_argv):

    # Set parameters.
    hp = HParams({
            flag.name: flag.value for flag in FLAGS.get_flags_for_module('__main__')
        })

    # Go through each sub-directory. Collect the last epoch val acc and test acc. Collect -1 and min_c accs.
    val_mean = np.zeros((1, 2)); val_std = np.zeros((1, 2));
    test_mean = np.zeros((1, 2)); test_std = np.zeros((1, 2));
    subdir_array = []
    for subdir in os.listdir(hp.directory):
        if subdir.endswith(".txt"):
            continue

        #choose = ('learning_rate_1e-05' in subdir) and ('weight_decay_0.1' in subdir) and ('stepsize_0.001' in subdir)
        #if not choose:
        #    continue

        #if 'WorstoffDRO' in subdir and not ('stepsize_0.01' in subdir):
        # continue

        val_item_all = []; val_item_min = [];
        test_item_all = []; test_item_min = [];
        for seed in hp.seed_list:

            csv_path = os.path.join(hp.directory, subdir, f'run_{seed}', 'stats', 'stats.csv')
            if not os.path.isfile(csv_path):
                csv_path = os.path.join(hp.directory, subdir, f'run_{seed}', 'stats.csv')

            try:
                data = np.genfromtxt(csv_path, dtype=None, delimiter=',', names=True, deletechars="")
            except:
                #print(f'\n\n TRY ERROR - {csv_path}\n\n')
                continue

            if len(data) < 1:
                #print(f'\n\n RUN ERROR - {subdir} did not finish run\n\n')
                continue

            # Picking the last epoch measurements
            val_item_all.append(data[f'val.acc.-1'][-1]); val_item_min.append(data[f'val.acc.{hp.min_c}'][-1]);
            test_item_all.append(data[f'test.acc.-1'][-1]); test_item_min.append(data[f'test.acc.{hp.min_c}'][-1]);

        # Place inside a numpy nd array and keep appending the numpy nd array.
        val_mean = np.append(val_mean, values=[[np.mean(val_item_all), np.mean(val_item_min)]], axis=0);
        val_std = np.append(val_std, values=[[np.std(val_item_all), np.std(val_item_min)]], axis=0);
        test_mean = np.append(test_mean, values=[[np.mean(test_item_all), np.mean(test_item_min)]], axis=0);
        test_std = np.append(test_std, values=[[np.std(test_item_all), np.std(test_item_min)]], axis=0);
        subdir_array.append(subdir)

    val_mean = np.delete(val_mean, (0), axis=0); val_std = np.delete(val_std, (0), axis=0);
    test_mean = np.delete(test_mean, (0), axis=0); test_std = np.delete(test_std, (0), axis=0);

    if 'CelebA' in hp.directory:
        print(val_mean.shape, val_std.shape,  test_mean.shape, test_std.shape)
        val_mean = val_mean[~np.isnan(val_mean).any(axis=1)]
        val_std = val_std[~np.isnan(val_std).any(axis=1)]
        test_mean = test_mean[~np.isnan(test_mean).any(axis=1)]
        test_std = test_std[~np.isnan(test_std).any(axis=1)]
        if (np.isnan(test_mean).any(axis=1)).sum() > 1:
            subdir_array = np.ones(len(test_std))
    print(val_mean.shape, val_std.shape,  test_mean.shape, test_std.shape)

    #print(repr(test_mean[:, 0]))
    #print(repr(test_mean[:, 1]))
    #pdb.set_trace()


    # Assign index
    acc_idx = 0
    min_c_idx = 1

    # Perform a group-sort as per the validation accuracy
    sort_idx = val_mean[:, acc_idx].argsort()[::-1]
    val_mean_sorted = val_mean[sort_idx]
    val_std_sorted = val_std[sort_idx]
    test_mean_sorted = test_mean[sort_idx]
    test_std_sorted = test_std[sort_idx]
    subdir_array_sorted = [subdir_array[idx] for idx in sort_idx]

    '''
    print(hp.directory)
    print(repr(test_mean_sorted[:, 0]))
    print(repr(test_mean_sorted[:, 1]))
    print(subdir_array_sorted)
    print(' ')
    '''

    # Among top hp.nvp_th accuracies, pick the highest min_c accuracy. Pick corresponding test and test_min_c accuracy
    select_idx = val_mean_sorted[:hp.nvp_th, min_c_idx].argmax()
    select_val_mean = val_mean_sorted[select_idx]
    select_val_std = val_std_sorted[select_idx]
    select_test_mean = test_mean_sorted[select_idx]
    select_test_std = test_std_sorted[select_idx]
    select_subdir = subdir_array_sorted[select_idx]

    '''
    # Among accuracies>hp.nvp_percent  of best accuracy, pick the highest min_c accuracy.
    # Pick corresponding test and test_min_c accuracy
    nvp_acc = 0.01*hp.nvp_percent*val_mean_sorted[0, acc_idx]
    nvp_choose = val_mean_sorted[:, acc_idx] > nvp_acc
    print('selects:', sum(nvp_choose))
    select_idx = val_mean_sorted[nvp_choose, min_c_idx].argmax() # since sorted array, index not corrupted
    select_val_mean = val_mean_sorted[select_idx]
    select_val_std = val_std_sorted[select_idx]
    select_test_mean = test_mean_sorted[select_idx]
    select_test_std = test_std_sorted[select_idx]
    select_subdir = subdir_array_sorted[select_idx]
    '''

    # Print the result
    string_to_print = f'File: {select_subdir}\n'
    string_to_print += f'VALIDATION-- Accuracy: {select_val_mean[acc_idx]:.4f} +- {select_val_std[acc_idx]:.4f} ||||          MinAccuracy: {select_val_mean[min_c_idx]:.4f} +- {select_val_std[min_c_idx]:.4f}\n'
    string_to_print += f'TESTING-- Accuracy: {select_test_mean[acc_idx]:.4f} +- {select_test_std[acc_idx]:.4f} ||||          MinAccuracy: {select_test_mean[min_c_idx]:.4f} +- {select_test_std[min_c_idx]:.4f}\n'
    print(string_to_print)

    ## log the result in tune.txt
    #with open(os.path.join(hp.directory, 'tune.txt'), 'w') as f:
    #    f.write(string_to_print)

if __name__ == '__main__':
    app.run(main)