-
Notifications
You must be signed in to change notification settings - Fork 0
/
process_dstat_baseline_2midd.py
136 lines (120 loc) · 5.64 KB
/
process_dstat_baseline_2midd.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
"""
ASL project - fall 2017
author: Jovan Nikolic
Processes aggregated logs generated by middleware
"""
import numpy as np
import csv
base_path_read = "data/baseline_2midd/dstat_"
base_path_write = "data/baseline_2midd/dstat_"
output_base_path = "plots/baseline_2midd/"
machine_types = ["memtier", "memcached"]
virtual_clients_pt = [1, 5, 8, 15, 22, 28, 32, 42, 52, 64]
worker_threads = [8, 16, 32, 64]
commands = ["1", "0"]
columns = ["user cpu", "system cpu", "net_received", "net_sent", "system csw"]
repetitions = 3
def read_csv(machine_type, cpt, wt, command):
full_data = {}
for column in columns:
full_data[column] = []
if machine_type == machine_types[0]:
max_id = 1
elif machine_type == machine_types[1]:
max_id = 1
else:
max_id = 0
if command == commands[0]:
base_path = base_path_read
else:
base_path = base_path_write
for rep in range(repetitions):
full_data_pr = {}
for column in columns:
full_data_pr[column] = []
current_rep = rep + 1
print(" rep: " + str(current_rep))
for i in range(max_id):
ID = i + 1
print(" id: " + str(ID))
path = base_path + machine_type + str(ID) + "_cpt" + str(cpt) + "_wt" + str(wt) + "_rep" + str(current_rep) + "_" + command + ".txt"
with open(path, 'r') as file:
data = file.readlines()
data = [x.strip() for x in data]
for k, line in enumerate(data):
if k < 7:
continue
parsed_line = line.split(',')
[x.strip() for x in parsed_line]
full_data_pr[columns[0]].append(float(parsed_line[0]))
full_data_pr[columns[1]].append(float(parsed_line[1]))
full_data_pr[columns[2]].append(float(parsed_line[18]))
full_data_pr[columns[3]].append(float(parsed_line[19]))
full_data_pr[columns[4]].append(float(parsed_line[25]))
full_data[columns[0]].append(np.mean(np.asarray(full_data_pr[columns[0]])))
full_data[columns[1]].append(np.mean(np.asarray(full_data_pr[columns[1]])))
full_data[columns[2]].append(np.mean(np.asarray(full_data_pr[columns[2]])))
full_data[columns[3]].append(np.mean(np.asarray(full_data_pr[columns[3]])))
full_data[columns[4]].append(np.mean(np.asarray(full_data_pr[columns[4]])))
full_data_means = {}
full_data_std = {}
for z, column in enumerate(columns):
if z == 2 or z == 3:
full_data_means[column] = np.mean(np.asarray(full_data[column])/1024)
full_data_std[column] = np.std(np.asarray(full_data[column])/1024)
else:
full_data_means[column] = np.mean(np.asarray(full_data[column]))
full_data_std[column] = np.std(np.asarray(full_data[column]))
return full_data_means, full_data_std
def print_csv(header, path, full_data_means, full_data_stds):
with open(path, 'w') as csv_file:
writer = csv.DictWriter(csv_file, fieldnames=header)
writer.writeheader()
for row in range(len(virtual_clients_pt)):
one_row = {}
i = 0
one_row[header[i]] = 2*virtual_clients_pt[row]
i += 1
for k in range(len(columns)):
one_row[header[i]] = full_data_means[commands[0]][virtual_clients_pt[row]][columns[k]]
i += 1
one_row[header[i]] = full_data_stds[commands[0]][virtual_clients_pt[row]][columns[k]]
i += 1
one_row[header[i]] = full_data_means[commands[1]][virtual_clients_pt[row]][columns[k]]
i += 1
one_row[header[i]] = full_data_stds[commands[1]][virtual_clients_pt[row]][columns[k]]
i += 1
writer.writerow(one_row)
csv_file.close()
def process():
for wt in worker_threads:
for machine_type in machine_types:
print("Machine: " + machine_type)
full_data_means = {}
full_data_stds = {}
for command in commands:
print(" Command: " + command)
full_data_means[command] = {}
full_data_stds[command] = {}
for cpt in virtual_clients_pt:
print(" Clients: " + str(cpt))
means, stds = read_csv(machine_type, cpt, wt, command)
full_data_means[command][cpt] = means
full_data_stds[command][cpt] = stds
header = ["#Clients",
"Mean User CPU - READ-ONLY", "Std User CPU - READ-ONLY",
"Mean User CPU - WRITE-ONLY", "Std User CPU - WRITE-ONLY",
"Mean System CPU - READ-ONLY", "Std System CPU - READ-ONLY",
"Mean System CPU - WRITE-ONLY", "Std System CPU - WRITE-ONLY",
"Mean Net-Received - READ-ONLY", "Std Net-Received - READ-ONLY",
"Mean Net-Received - WRITE-ONLY", "Std Net-Received - WRITE-ONLY",
"Mean Net-Sent - READ-ONLY", "Std Net-Sent - READ-ONLY",
"Mean Net-Sent - WRITE-ONLY", "Std Net-Sent - WRITE-ONLY",
"Mean Context Switches - READ-ONLY", "Std Context Switches - READ-ONLY",
"Mean Context Switches - WRITE-ONLY", "Std Context Switches - WRITE-ONLY"]
path = output_base_path + "dstat_" + machine_type + "_wt" + str(wt) + ".csv"
print_csv(header, path, full_data_means, full_data_stds)
def main():
process()
if __name__ == "__main__":
main()