-
Notifications
You must be signed in to change notification settings - Fork 66
/
results_display.py
128 lines (121 loc) · 5.03 KB
/
results_display.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
from __init__ import *
from keras.models import load_model
key = 'artificial'
mindate = '2017-02-13'
files = [f for f in os.listdir('results') if (key in f) and ('.pkl' in f)]
read_tables = []
for f in files:
try:
df = pd.read_pickle(WDIR + '/results/' + f)
print('1, FILE ' + f + ' SHAPE ' + repr(df.shape))
df = df[df['dt'] > pd.Timestamp(mindate)]
df['epochs'] = df['loss'].apply(lambda x: len(x))
print('2, FILE ' + f + ' SHAPE ' + repr(df.shape))
df = df[df['epochs'] > 5]
print('3, FILE ' + f + ' SHAPE ' + repr(df.shape))
if df.shape[0] == 0:
continue
if 'objective' not in df.columns:
df['objective'] = 'regr' if ('regr' in f) else 'class'
if 'returns' not in df.columns:
df['returns'] = ('ret' in f)
elif 'both' in f:
df = df[df['returns']] ###### screwed up classification loss in 'both'
if 'full_shift' not in df.columns:
df['full_shift'] = True
df['NeX'] = ('NeX' in f)
if 'batch_size' not in df.columns:
df['batch_size'] = 64
if 'channels' not in df.columns:
if 'filters' in df.columns:
df['channels'] = df['filters']
else:
df['channels'] = 1
else:
df.loc[np.isnan(df['channels']), 'channels'] = 1
if 'model' in df.columns:
pass
elif ('benchmark' in f) or ('lr' in f):
df['model'] = 'lr'
elif 'cnn' in f:
df['model'] = 'cnn'
elif 'gru' in f:
df['model'] = 'gru'
elif 'lstm' in f:
df['model'] = 'lstm'
elif 'value_importance3' in f:
df['model'] = 'so3'
elif ('cvi2' in f) or ('cnnvi2' in f):
df['model'] = 'cvi2'
elif ('cvi3' in f) or ('cnnvi3' in f):
df['model'] = 'cvi3'
elif ('cvi' in f) or ('cnnvi' in f):
df['model'] = 'cvi'
else:
df['model'] = 'so'
keys, funcs, names = [], [], []
df['best_mse'] = np.NaN
df['best_acc'] = np.NaN
if ('class' in f) or all(df['objective'] == 'class'):
if 'val_main_output_acc' in df.columns:
keys.append('val_main_output_acc')
funcs.append([np.nanmax, np.nanargmax, [-np.inf]])
names.append('best_acc')
elif 'val_acc' in df.columns:
keys.append('val_acc')
funcs.append([np.nanmax, np.nanargmax, [-np.inf]])
names.append('best_acc')
if ('regr' in f) or all(df['objective'] == 'regr') :
if 'val_main_output_loss' in df.columns:
keys.append('val_main_output_loss')
funcs.append([np.nanmin, np.nanargmin, [np.inf]])
names.append('best_mse')
elif 'val_loss' in df.columns:
keys.append('val_loss')
funcs.append([np.nanmin, np.nanargmin, [np.inf]])
names.append('best_mse')
for k, func, n in zip(keys, funcs, names):
df[n] = df[k].apply(lambda x: func[0](x + func[2]))
df['best_epoch'] = df[k].apply(lambda x: func[1](x + func[2]))
df['file'] = ''.join(f.split(key)) if (len(key) > 0) else f
df['time_per_epoch'] = df['training_time'] / df['epochs']
df['time_to_best'] = df['best_epoch'] * df['time_per_epoch']
read_tables.append(df.copy())
except Exception as e:
print(e)
def total_params(model):
return np.sum([np.sum([np.prod(K.eval(w).shape) for w in l.trainable_weights]) for l in model.layers])
all_results = pd.concat(read_tables)
all_results.reset_index(inplace=True)
# if 'total_params' not in all_results.columns:
# all_results['total_params'] = all_results['hdf5'].apply(lambda x: total_params(load_model(x)))
# else:
# idx = np.isnan(all_results['total_params'])
# all_results.loc[idx, 'total_params'] = all_results.loc[idx, 'hdf5'].apply(lambda x: total_params(load_model(x)))
all_results.to_pickle(WDIR + '/results/all_results.pkl')
def get_pivot(setting, df, valcols=[]):
if setting['objective'] == 'class':
best = 'best_acc'
func = max
else:
best = 'best_mse'
func = min
df0 = df.copy()
for k, v in setting.items():
df0 = df0[df0[k] == v]
df0 = df0[df0.groupby(['data', 'model'])[best].transform(func) == df0[best]]
pivs = [df0.pivot(index='data', columns='model', values=best)]
for col in valcols:
piv = df0.pivot(index='data', columns='model', values=col)
piv.rename(columns=dict([(c, col[:3] + '_' + c) for c in piv.columns]), inplace=True)
pivs.append(piv.copy())
return df0, pd.concat(pivs, axis=1)
print(all_results.groupby('file').count())
setting = dict(
objective = 'regr',
train_share = (.8, 1.),
diffs = False,
# architecture = {'lambda': True, 'softmax':False, 'nonneg':False}
)
df0, pivot = get_pivot(setting, all_results, ['total_params', 'time_to_best'])
print(pivot)