-
Notifications
You must be signed in to change notification settings - Fork 20
/
eval.py
187 lines (148 loc) · 7.27 KB
/
eval.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
"""
Author: Talip Ucar
email: ucabtuc@gmail.com
Description: Wrapper function for evaluation routine.
"""
import mlflow
import torch as th
import torch.utils.data
from tqdm import tqdm
from src.model import SubTab
from utils.arguments import get_arguments, get_config
from utils.arguments import print_config_summary
from utils.eval_utils import linear_model_eval, plot_clusters, append_tensors_to_lists, concatenate_lists, aggregate
from utils.load_data import Loader
from utils.utils import set_dirs, run_with_profiler, update_config_with_model_dims
torch.manual_seed(1)
def eval(data_loader, config):
"""Wrapper function for evaluation.
Args:
data_loader (IterableDataset): Pytorch data loader.
config (dict): Dictionary containing options and arguments.
"""
# Instantiate Autoencoder model
model = SubTab(config)
# Load the model
model.load_models()
# Evaluate Autoencoder
with th.no_grad():
# Get the joint embeddings and class labels of training set
z_train, y_train = evalulate_models(data_loader, model, config, plot_suffix="training", mode="train")
# Train and evaluate logistig regression using the joint embeddings of training and test set
evalulate_models(data_loader, model, config, plot_suffix="test", mode="test", z_train=z_train, y_train=y_train)
# End of the run
print(f"Evaluation results are saved under ./results/{config['framework']}/evaluation/\n")
print(f"{100 * '='}\n")
# If mlflow==True, track results
if config["mlflow"]:
# Log model and results with mlflow
mlflow.log_artifacts(model._results_path + "/evaluation/" + "/clusters", "evaluation")
def evalulate_models(data_loader, model, config, plot_suffix="_Test", mode='train', z_train=None, y_train=None):
"""Evaluates representations using linear model, and visualisation of clusters using t-SNE and PCA on embeddings.
Args:
data_loader (IterableDataset): Pytorch data loader.
model (object): Class that contains the encoder and associated methods
config (dict): Dictionary containing options and arguments.
plot_suffix (str): Suffix to be used when saving plots
mode (str): Defines whether to evaluate the model on training set, or test set.
z_train (ndarray): Optional numpy array holding latent representations of training set
y_train (list): Optional list holding labels of training set
Returns:
(tuple): tuple containing:
z_train (numpy.ndarray): Numpy array holding latent representations of data set
y_train (list): List holding labels of data set
"""
# A small function to print a line break on the command line.
break_line = lambda sym: f"{100 * sym}\n{100 * sym}\n"
# Print whether we are evaluating training set, or test set
decription = break_line('#') + f"Getting the joint embeddings of {plot_suffix} set...\n" + \
break_line('=') + f"Dataset used: {config['dataset']}\n" + break_line('=')
# Print the message
print(decription)
# Get the model
encoder = model.encoder
# Move the model to the device
encoder.to(config["device"])
# Set the model to evaluation mode
encoder.eval()
# Choose either training, or test data loader
data_loader_tr_or_te = data_loader.train_loader if mode == 'train' else data_loader.test_loader
# Attach progress bar to data_loader to check it during training. "leave=True" gives a new line per epoch
train_tqdm = tqdm(enumerate(data_loader_tr_or_te), total=len(data_loader_tr_or_te), leave=True)
# Create empty lists to hold data for representations, and class labels
z_l, clabels_l = [], []
# Go through batches
for i, (x, label) in train_tqdm:
# Generate subsets
x_tilde_list = model.subset_generator(x)
latent_list = []
# Extract embeddings (i.e. latent) for each subset
for xi in x_tilde_list:
# Turn xi to tensor, and move it to the device
Xbatch = model._tensor(xi)
# Extract latent
_, latent, _ = encoder(Xbatch)
# Collect latent
latent_list.append(latent)
# Aggregation of latent representations
latent = aggregate(latent_list, config)
# Append tensors to the corresponding lists as numpy arrays
z_l, clabels_l = append_tensors_to_lists([z_l, clabels_l],
[latent, label.int()])
# Turn list of numpy arrays to a single numpy array for representations.
z = concatenate_lists([z_l])
# Turn list of numpy arrays to a single numpy array for class labels.
clabels = concatenate_lists([clabels_l])
# Visualise clusters
plot_clusters(config, z, clabels, plot_suffix="_inLatentSpace_" + plot_suffix)
if mode == 'test':
# Title of the section to print
print(20 * "*" + " Running evaluation using Logistic Regression trained on the joint embeddings" \
+ " of training set and tested on that of test set" + 20 * "*")
# Description of the task (Classification scores using Logistic Regression) to print on the command line
description = "Sweeping C parameter. Smaller C values specify stronger regularization:"
# Evaluate the embeddings
linear_model_eval(config, z_train, y_train, z_test=z, y_test=clabels, description=description)
else:
# Return z_train = z, and y_train = clabels
return z, clabels
def main(config):
"""Main function for evaluation
Args:
config (dict): Dictionary containing options and arguments.
"""
# Set directories (or create if they don't exist)
set_dirs(config)
# Get data loader for first dataset.
ds_loader = Loader(config, dataset_name=config["dataset"], drop_last=False)
# Add the number of features in a dataset as the first dimension of the model
config = update_config_with_model_dims(ds_loader, config)
# Start evaluation
eval(ds_loader, config)
if __name__ == "__main__":
# Get parser / command line arguments
args = get_arguments()
# Get configuration file
config = get_config(args)
# Overwrite the parent folder name for saving results
config["framework"] = config["dataset"]
# Turn off valiation
config["validate"] = False
# Get all of available training set for evaluation (i.e. no need for validation set)
config["training_data_ratio"] = 1.0
# Turn off noise when evaluating the performance
config["add_noise"] = False
# Summarize config and arguments on the screen as a sanity check
print_config_summary(config, args)
# --If True, start of MLFlow for experiment tracking:
if config["mlflow"]:
# Experiment name
experiment_name = "Give_Your_Experiment_A_Name"
mlflow.set_experiment(experiment_name=experiment_name + "_" + str(args.experiment))
# Start a new mlflow run
with mlflow.start_run():
# Run the main with or without profiler
run_with_profiler(main, config) if config["profile"] else main(config)
else:
# Run the main with or without profiler
run_with_profiler(main, config) if config["profile"] else main(config)