process_aggregated_data_timers_experiment_gets.py

"""
     ASL project - fall 2017

        author: Jovan Nikolic

        Processes aggregated logs generated by middleware
"""
import numpy as np
import csv

agg_path_base = "aggregated_data/experiment_gets/timers_2/"
plot_path_base = "plots/experiment_gets/timers_2/"
name_base = "timer_aggregated_data_"
client_threads_basename = "clientThreads_"
worker_threads_basename = "_workerThreads_"
counters_basename = "counter_"
timers_basename = "timers_"

number_of_middlewares = 2
cpt = 2
wt = 64
num_keys = [1, 3, 6, 9]
suffixes = ["sharded", "nonsharded"]
metrics = ["mean", "std"]
repetitions = 3

memtier_vms = 3
memtier_instances_per_vm = 2
memtier_threads_per_inst = 1


def read_csv(keys, index):
    response_time = {}
    net_thread_processing_time = {}
    waiting_in_queue_time = {}
    worker_preprocessing_time = {}
    server_service_time = {}
    worker_postprocessing_time = {}
    queue_size = {}
    beginning_of_time = {}
    for mw in range(number_of_middlewares):
        current_mw = mw + 1

        final_agg_path = agg_path_base + "timer_aggregated_data_" + \
                         "mw_" + str(current_mw) +\
                         client_threads_basename + str(cpt) + \
                         worker_threads_basename + str(wt) + \
                         "_" + suffixes[index] + "_keys" + str(keys) + ".csv"

        response_time[mw] = {}
        net_thread_processing_time[mw] = {}
        waiting_in_queue_time[mw] = {}
        worker_preprocessing_time[mw] = {}
        server_service_time[mw] = {}
        worker_postprocessing_time[mw] = {}
        queue_size[mw] = {}
        beginning_of_time[mw] = {}

        for rep in range(repetitions):
            a = []
            response_time[mw][rep] = a
            b = []
            net_thread_processing_time[mw][rep] = b
            c = []
            waiting_in_queue_time[mw][rep] = c
            d = []
            worker_preprocessing_time[mw][rep] = d
            e = []
            server_service_time[mw][rep] = e
            f = []
            worker_postprocessing_time[mw][rep] = f
            g = []
            queue_size[mw][rep] = g

        with open(final_agg_path, 'r') as file:
            timer_data = file.readlines()
            timer_data = [x.strip() for x in timer_data]
            for k, line in enumerate(timer_data):
                if k == 0:
                    continue
                parsed_line = line.split(',')
                [x.strip() for x in parsed_line]

                column = 1

                if k == 1:
                    for rep in range(repetitions):
                        beginning_of_time[mw][rep] = float(parsed_line[column])
                        column += 1
                else:
                    column = repetitions + 1

                for rep in range(repetitions):
                    response_time[mw][rep].append(float(parsed_line[column]))
                    column += 1
                for rep in range(repetitions):
                    net_thread_processing_time[mw][rep].append(float(parsed_line[column]))
                    column += 1
                for rep in range(repetitions):
                    waiting_in_queue_time[mw][rep].append(float(parsed_line[column]))
                    column += 1
                for rep in range(repetitions):
                    worker_preprocessing_time[mw][rep].append(float(parsed_line[column]))
                    column += 1
                for rep in range(repetitions):
                    server_service_time[mw][rep].append(float(parsed_line[column]))
                    column += 1
                for rep in range(repetitions):
                    worker_postprocessing_time[mw][rep].append(float(parsed_line[column]))
                    column += 1
                for rep in range(repetitions):
                    queue_size[mw][rep].append(float(parsed_line[column]))
                    column += 1

    # cut_left = 10
    # cut_right = 89
    #
    # if worker_thread == 8:
    #     cut_left = 5
    #     cut_right = 80

    cut_left = 10
    cut_right = 80

    response_time_pr = {}
    net_thread_processing_time_pr = {}
    waiting_in_queue_time_pr = {}
    worker_preprocessing_time_pr = {}
    server_service_time_pr = {}
    worker_postprocessing_time_pr = {}
    queue_size_pr = {}

    for rep in range(repetitions):
        full_list = []
        for mw in range(number_of_middlewares):
            print("MW = " + str(mw) + ", rep = " + str(rep) + ", len = " )
            full_list = np.concatenate([np.asarray(full_list), response_time[mw][rep]])
        full_list_sorted = sorted(full_list)
        mean_val = np.mean(np.asarray(full_list_sorted)[cut_left:cut_right])
        response_time_pr[rep] = mean_val

        full_list = []
        for mw in range(number_of_middlewares):
            full_list = np.concatenate([np.asarray(full_list), net_thread_processing_time[mw][rep]])
        full_list_sorted = sorted(full_list)
        mean_val = np.mean(np.asarray(full_list_sorted)[cut_left:cut_right])
        net_thread_processing_time_pr[rep] = mean_val

        full_list = []
        for mw in range(number_of_middlewares):
            full_list = np.concatenate([np.asarray(full_list), waiting_in_queue_time[mw][rep]])
        full_list_sorted = sorted(full_list)
        mean_val = np.mean(np.asarray(full_list_sorted)[cut_left:cut_right])
        waiting_in_queue_time_pr[rep] = mean_val

        full_list = []
        for mw in range(number_of_middlewares):
            full_list = np.concatenate([np.asarray(full_list), worker_preprocessing_time[mw][rep]])
        full_list_sorted = sorted(full_list)
        mean_val = np.mean(np.asarray(full_list_sorted)[cut_left:cut_right])
        worker_preprocessing_time_pr[rep] = mean_val

        full_list = []
        for mw in range(number_of_middlewares):
            full_list = np.concatenate([np.asarray(full_list), server_service_time[mw][rep]])
        full_list_sorted = sorted(full_list)
        mean_val = np.mean(np.asarray(full_list_sorted)[cut_left:cut_right])
        server_service_time_pr[rep] = mean_val

        full_list = []
        for mw in range(number_of_middlewares):
            full_list = np.concatenate([np.asarray(full_list), worker_postprocessing_time[mw][rep]])
        full_list_sorted = sorted(full_list)
        mean_val = np.mean(np.asarray(full_list_sorted)[cut_left:cut_right])
        worker_postprocessing_time_pr[rep] = mean_val

        full_list = []
        for mw in range(number_of_middlewares):
            full_list = np.concatenate([np.asarray(full_list), queue_size[mw][rep]])
        full_list_sorted = sorted(full_list)
        mean_val = np.mean(np.asarray(full_list_sorted)[cut_left:cut_right])
        queue_size_pr[rep] = mean_val

    # now we calculate the mean between repetitions and standard deviation between them

    array = []
    for rep in range(repetitions):
        array.append(response_time_pr[rep])
    response_time_mean = np.mean(array)
    response_time_std = np.std(array)

    array = []
    for rep in range(repetitions):
        array.append(net_thread_processing_time_pr[rep])
    net_thread_processing_time_mean = np.mean(array)
    net_thread_processing_time_std = np.std(array)

    array = []
    for rep in range(repetitions):
        array.append(waiting_in_queue_time_pr[rep])
    waiting_in_queue_time_mean = np.mean(array)
    waiting_in_queue_time_std = np.std(array)

    array = []
    for rep in range(repetitions):
        array.append(worker_preprocessing_time_pr[rep])
    worker_preprocessing_time_mean = np.mean(array)
    worker_preprocessing_time_std = np.std(array)

    array = []
    for rep in range(repetitions):
        array.append(server_service_time_pr[rep])
    server_service_time_mean = np.mean(array)
    server_service_time_std = np.std(array)

    array = []
    for rep in range(repetitions):
        array.append(worker_postprocessing_time_pr[rep])
    worker_postprocessing_time_mean = np.mean(array)
    worker_postprocessing_time_std = np.std(array)

    array = []
    for rep in range(repetitions):
        array.append(queue_size_pr[rep])
    queue_size_mean = np.mean(array)
    queue_size_std = np.std(array)

    return [response_time_mean, response_time_std], [net_thread_processing_time_mean, net_thread_processing_time_std], [waiting_in_queue_time_mean, waiting_in_queue_time_std], [worker_preprocessing_time_mean, worker_preprocessing_time_std], [server_service_time_mean, server_service_time_std], [worker_postprocessing_time_mean, worker_postprocessing_time_std], [queue_size_mean, queue_size_std], beginning_of_time
    # return [response_time_mean, response_time_std], [net_thread_processing_time_mean, net_thread_processing_time_std], [
    #     waiting_in_queue_time_mean, waiting_in_queue_time_std], [worker_preprocessing_time_mean,
    #                                                              worker_preprocessing_time_std], [
    #            server_service_time_mean, server_service_time_std], [worker_postprocessing_time_mean,
    #                                                                 worker_postprocessing_time_std], [queue_size_mean,
    #                                                                                                   queue_size_std], [
    #            beginning_of_time[0]]


def print_csv(path, header, full_data):
    print("Header length is: " + str(len(header)))
    print("Number of rows is: " + str(len(full_data)))
    print("NUmber of columns is: " + str(len(full_data[0])))

    with open(path, 'w') as csv_file:
        writer = csv.DictWriter(csv_file, fieldnames=header)
        writer.writeheader()

        for row in range(len(full_data)):
            one_row = {}
            for i in range(len(header)):
                one_row[header[i]] = full_data[row][i]
            writer.writerow(one_row)
        csv_file.close()


def print_all_times(suffix, header, response_time, net_thread_processing_time,
                    wait_in_queue_time, worker_preprocessing_time, server_service_time,
                    worker_postprocessing_time):
    full_data = []

    for row in range(len(response_time)):
        row_data = [response_time[row][0], net_thread_processing_time[row][1], wait_in_queue_time[row][1],
                 worker_preprocessing_time[row][1], server_service_time[row][1], worker_postprocessing_time[row][1],
                 response_time[row][1], response_time[row][2]]
        full_data.append(row_data)

    path = plot_path_base + "all_times_" + suffix + ".csv"
    with open(path, 'w') as the_file:
        the_file.write(header)
        for row in range(len(full_data)):
            row_data = str(int(full_data[row][0]))
            for k in range(1, len(full_data[row])):
                row_data += "\t" + str(full_data[row][k])
            row_data += "\n"
            the_file.write(row_data)
        the_file.close()


def main():
    for index, suffix in enumerate(suffixes):
        response_time = []
        net_thread_processing_time = []
        waiting_in_queue_time = []
        worker_preprocessing_time = []
        server_service_time = []
        worker_postprocessing_time = []
        queue_size = []

        for keys in num_keys:

            response_time_row = [keys]
            net_thread_processing_time_row = [keys]
            waiting_in_queue_time_row = [keys]
            worker_preprocessing_time_row = [keys]
            server_service_time_row = [keys]
            worker_postprocessing_time_row = [keys]
            queue_size_row = [keys]

            a, b, c, d, e, f, g, h = read_csv(keys, index)
            response_time_row = np.concatenate([np.asarray(response_time_row), a])

            net_thread_processing_time_row = np.concatenate([np.asarray(net_thread_processing_time_row), b])
            waiting_in_queue_time_row = np.concatenate([np.asarray(waiting_in_queue_time_row), c])
            worker_preprocessing_time_row = np.concatenate([np.asarray(worker_preprocessing_time_row), d])
            server_service_time_row = np.concatenate([np.asarray(server_service_time_row), e])
            worker_postprocessing_time_row = np.concatenate([np.asarray(worker_postprocessing_time_row), f])
            queue_size_row = np.concatenate([np.asarray(queue_size_row), g])

            response_time.append(response_time_row)
            net_thread_processing_time.append(net_thread_processing_time_row)
            waiting_in_queue_time.append(waiting_in_queue_time_row)
            worker_preprocessing_time.append(worker_preprocessing_time_row)
            server_service_time.append(server_service_time_row)
            worker_postprocessing_time.append(worker_postprocessing_time_row)
            queue_size.append(queue_size_row)

        header = ["#Keys", "Mean Response Time [ms]", "Std Response Time"]
        path = plot_path_base + "response_time_" + suffix + ".csv"
        print_csv(path, header, response_time)

        header = ["#Keys", "Mean NetThread Processing Time [ms]", "Std NetThread Processing Time"]
        path = plot_path_base + "netthread_processing_time_" + suffix + ".csv"
        print_csv(path, header, net_thread_processing_time)

        header = ["#Keys", "Mean Wait-In-Queue Time [ms]", "Std Wait-In-Queue Time"]
        path = plot_path_base + "wait_in_queue_time_" + suffix + ".csv"
        print_csv(path, header, waiting_in_queue_time)

        header = ["#Keys", "Mean Pre-Processing Time [ms]", "Std Pre-Processing Time"]
        path = plot_path_base + "worker_preprocessing_time_" + suffix + ".csv"
        print_csv(path, header, worker_preprocessing_time)

        header = ["#Keys", "Mean Server Service Time [ms]", "Std Server Service Time"]
        path = plot_path_base + "server_service_time_" + suffix + ".csv"
        print_csv(path, header, server_service_time)

        header = ["#Keys", "Mean Post-Processing Time [ms]", "Std Post-Processing Time"]
        path = plot_path_base + "worker_postprocessing_time_" + suffix + ".csv"
        print_csv(path, header, worker_postprocessing_time)

        header = ["#Keys", "Mean Queue Size", "Std Queue Size"]
        path = plot_path_base + "queue_size_" + suffix + ".csv"
        print_csv(path, header, queue_size)

        header = "#Keys   MNT WIQ PrP SST PsP RspT    RespT_std\n"
        print_all_times(suffix, header, response_time, net_thread_processing_time, waiting_in_queue_time,
                        worker_preprocessing_time, server_service_time, worker_postprocessing_time)


if __name__ == "__main__":
    main()