Skip to content

Commit

Permalink
feat: save validation result as json and show the static dashboard us…
Browse files Browse the repository at this point in the history
…ing js

Signed-off-by: Parul Singh <parsingh@redhat.com>
  • Loading branch information
husky-parul committed Aug 7, 2024
1 parent c607a6d commit 6a6017b
Show file tree
Hide file tree
Showing 6 changed files with 366 additions and 9 deletions.
41 changes: 41 additions & 0 deletions e2e/tools/validator/html/index.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Chart with JSON Data</title>
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
<script defer src="script.js"></script>
<style>
body {
display: flex;
}
#chart-container {
width: 70%;
padding: 20px;
}
#info-container {
width: 30%;
padding: 20px;
}
ul {
list-style-type: none;
}
</style>
</head>
<body>
<div id="chart-container">
<canvas id="myChart"></canvas>
</div>
<div id="info-container">
<div id="build-info">
<h3>Build Info</h3>
<ul></ul>
</div>
<div id="machine-spec">
<h3>Machine Spec</h3>
<ul></ul>
</div>
</div>
</body>
</html>
128 changes: 128 additions & 0 deletions e2e/tools/validator/html/script.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
async function fetchJSONFile(filePath) {
try {
const response = await fetch(filePath);
const data = await response.json();
return data;
} catch (error) {
console.error('Error fetching JSON file:', error);
}
}

function processJSONData(data) {
const revisions = [];
const mseValues = [];
const mapeValues = [];
const labels = [];

data.build_info.forEach(buildInfo => {
const buildInfoObj = JSON.parse(buildInfo);
const revision = buildInfoObj.revision;
if (!revisions.includes(revision)) {
revisions.push(revision);
}
});

data.result.forEach(result => {
labels.push(result['metric-name']);
mseValues.push(parseFloat(result.value.mse));
mapeValues.push(parseFloat(result.value.mape));
});

return { revisions, mseValues, mapeValues, labels };
}

function plotChart(revisions, mseValues, mapeValues, labels) {
const ctx = document.getElementById('myChart').getContext('2d');
new Chart(ctx, {
type: 'line',
data: {
labels: labels,
datasets: [{
label: `MSE - ${revisions.join(', ')}`,
data: mseValues,
borderColor: 'rgba(75, 192, 192, 1)',
backgroundColor: 'rgba(75, 192, 192, 0.2)',
fill: false
}, {
label: `MAPE - ${revisions.join(', ')}`,
data: mapeValues,
borderColor: 'rgba(255, 99, 132, 1)',
backgroundColor: 'rgba(255, 99, 132, 0.2)',
fill: false
}]
},
options: {
responsive: true,
scales: {
x: {
beginAtZero: true
},
y: {
beginAtZero: true,
title: {
display: true,
text: 'Values'
}
}
},
plugins: {
legend: {
display: true,
position: 'top'
},
tooltip: {
callbacks: {
label: function(context) {
let label = context.dataset.label || '';
if (label) {
label += ': ';
}
if (context.parsed.y !== null) {
label += new Intl.NumberFormat('en-US', { maximumFractionDigits: 2 }).format(context.parsed.y);
}
return label;
}
}
}
}
}
});
}

function displayBuildInfo(data) {
const buildInfoContainer = document.getElementById('build-info').querySelector('ul');
data.build_info.forEach(info => {
const buildInfoObj = JSON.parse(info);
const listItem = document.createElement('li');
listItem.textContent = `Revision: ${buildInfoObj.revision}, Version: ${buildInfoObj.version}`;
buildInfoContainer.appendChild(listItem);
});
}

function displayMachineSpec(data) {
const machineSpecContainer = document.getElementById('machine-spec').querySelector('ul');
data.machine_spec.forEach(spec => {
const listItem = document.createElement('li');
listItem.innerHTML = `
<strong>Type:</strong> ${spec.type}<br>
<strong>Model:</strong> ${spec.model}<br>
<strong>Cores:</strong> ${spec.cores}<br>
<strong>Threads:</strong> ${spec.threads}<br>
<strong>Sockets:</strong> ${spec.sockets}<br>
<strong>DRAM:</strong> ${spec.dram}
`;
machineSpecContainer.appendChild(listItem);
});
}

// Path to the single JSON file
const filePath = '/tmp/v0.2-2315-g859d9bf1/v0.2-2316-gc607a6df.json'; // Change to the actual path of your JSON file

fetchJSONFile(filePath)
.then(data => {
const { revisions, mseValues, mapeValues, labels } = processJSONData(data);
plotChart(revisions, mseValues, mapeValues, labels);
displayBuildInfo(data);
displayMachineSpec(data);
})
.catch(error => console.error('Error processing data:', error));
23 changes: 23 additions & 0 deletions e2e/tools/validator/html/style.css
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
body {
font-family: Arial, sans-serif;
display: flex;
justify-content: center;
align-items: center;
height: 100vh;
margin: 0;
background-color: #f4f4f4;
}

#dashboard {
width: 80%;
max-width: 800px;
background: white;
padding: 20px;
box-shadow: 0 0 10px rgba(0, 0, 0, 0.1);
border-radius: 8px;
}

canvas {
max-width: 100%;
height: auto;
}
79 changes: 78 additions & 1 deletion e2e/tools/validator/src/validator/cli/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import json
import logging
import os
import re
import subprocess
import typing
from dataclasses import dataclass
Expand All @@ -16,12 +17,14 @@
from validator.__about__ import __version__
from validator.cli import options
from validator.prometheus import Comparator, PrometheusClient, Series, ValueOrError
from validator.report import CustomEncoder, JsonTemplate
from validator.specs import MachineSpec, get_host_spec, get_vm_spec
from validator.stresser import Remote, ScriptResult
from validator.validations import Loader, QueryTemplate, Validation

logger = logging.getLogger(__name__)
pass_config = click.make_pass_decorator(config.Validator)
data_dict = {}


@dataclass
Expand Down Expand Up @@ -272,6 +275,8 @@ def stress(cfg: config.Validator, script_path: str, report_dir: str):
click.secho(" * Generating report dir and tag", fg="green")
results_dir, tag = create_report_dir(report_dir)
click.secho(f"\tresults dir: {results_dir}, tag: {tag}", fg="bright_green")
filepath = results_dir + "/" + tag + ".json"
data_dict.update({"file_path": filepath})

res = TestResult(tag)

Expand All @@ -288,7 +293,7 @@ def stress(cfg: config.Validator, script_path: str, report_dir: str):
res.end_time = stress_test.end_time

res.validations = run_validations(cfg, stress_test, results_dir)

create_json(res)
write_md_report(results_dir, res)


Expand Down Expand Up @@ -441,3 +446,75 @@ def validate_acpi(cfg: config.Validator, duration: datetime.timedelta, report_di
write_md_report(results_dir, res)

return int(res.validations.passed)


def create_json(res):
def update_list_json(new_value: list, new_key: str):
data_dict[new_key] = new_value

def custom_encode(input_string):
pattern = re.compile(r'(\w+)=("[^"]*"|[^,]+)')
matches = pattern.findall(input_string)
parsed_dict = {key: value.strip('"') for key, value in matches}
return json.dumps(parsed_dict)

result = []

for i in res.validations.results:
value = {}
if i.mse_passed:
value["mse"] = float(i.mse.value)
else:
value["mse"] = float(i.mse.error)
if i.mape_passed:
value["mape"] = float(i.mape.value)
else:
value["mape"] = float(i.mape.error)
value["status"] = "mape passed: " + str(i.mape_passed) + ", mse passed: " + str(i.mse_passed)
m_name = i.name.replace(" - ", "_")

result.append({m_name: value})

build_info = []
for i in res.build_info:
tmp = i.replace("kepler_exporter_build_info", "")
build_info.append(custom_encode(tmp))

node_info = []
for i in res.node_info:
tmp = i.replace("kepler_exporter_node_info", "")
node_info.append(custom_encode(tmp))

update_list_json(build_info, "build_info")
update_list_json(node_info, "node_info")

machine_spec = []
machine_spec.append(
{
"type": "host",
"model": res.host_spec[0][0],
"cores": res.host_spec[0][1],
"threads": res.host_spec[0][2],
"sockets": res.host_spec[0][3],
"flags": res.host_spec[0][4],
"dram": res.host_spec[1],
}
)
machine_spec.append(
{
"type": "vm",
"model": res.vm_spec[0][0],
"cores": res.vm_spec[0][1],
"threads": res.vm_spec[0][2],
"sockets": res.vm_spec[0][3],
"flags": res.vm_spec[0][4],
"dram": res.vm_spec[1],
}
)
update_list_json(machine_spec, "machine_spec")

update_list_json(result, "result")
json_template = JsonTemplate(**data_dict)
file_name = data_dict["file_path"]
with open(file_name, "w") as file:
json.dump(json_template, file, cls=CustomEncoder, indent=2)
80 changes: 80 additions & 0 deletions e2e/tools/validator/src/validator/report/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
import json
from typing import Any


class Value:
def __init__(self, mse: str = "", mape: str = "", status: str = ""):
self.mse = mse
self.mape = mape
self.status = status

def to_dict(self):
return {"mse": self.mse, "mape": self.mape, "status": self.status}

def __repr__(self):
return f"Value(mse='{self.mse}', mape='{self.mape}', status='{self.status}')"


class Result:
def __init__(self, metric_name: str, value: dict[str, Any]):
if value is None:
value = {}
self.metric_name = metric_name
self.value = Value(**value)

def to_dict(self):
return {"metric-name": self.metric_name, "value": self.value.to_dict()}

def __repr__(self):
return f"Result(metric_name='{self.metric_name}', value={self.value})"


class JsonTemplate:
def __init__(
self,
file_path: str,
build_info: list[Any],
node_info: list[Any],
machine_spec: list[Any],
result: list[dict[str, Any]],
):
if build_info is None:
build_info = []
if node_info is None:
node_info = []
if machine_spec is None:
machine_spec = []
if result is None:
result = []

self.file_path = file_path
self.build_info = build_info
self.node_info = node_info
self.machine_spec = machine_spec
self.result = []
for res in result:
for key, value in res.items():
self.result.append(Result(key, value))

def to_dict(self):
return {
"file_path": self.file_path,
"build_info": self.build_info,
"node_info": self.node_info,
"machine_spec": self.machine_spec,
"result": [res.to_dict() for res in self.result],
}

def __repr__(self):
return (
f"JsonTemplate(file_path='{self.file_path}', build_info={self.build_info}, "
f"node_info={self.node_info}, machine_spec={self.machine_spec}, "
f"result={self.result})"
)


class CustomEncoder(json.JSONEncoder):
def default(self, obj):
if hasattr(obj, "to_dict"):
return obj.to_dict()
return super().default(obj)
Loading

0 comments on commit 6a6017b

Please sign in to comment.