metrics: report: storage: add fio test to reportgen

Add the fio test, in default mode, to the report gen
grabdata script storage section.

For reference, on my machine the default fio test (random
read and write in direct mode with a small set of blocksizes)
takes about 10minutes to run.

Signed-off-by: Graham Whaley <graham.whaley@intel.com>

metrics/report/README.md

@@ -27,6 +27,10 @@ Repeat this process if you want to compare multiple sets of results. Note, the
 report generation scripts process all subfolders of `tests/metrics/results` when
 generating the report.
 
+> **Note:** By default, the `grabdata.sh` script ties to launch some moderately
+> large containers (8Gbyte RAM), so may fail to produce some results on a memory
+> constrained system.
+
 ## Report generation
 
 Report generation is provided by the `makereport.sh` script. By default this script 

metrics/report/grabdata.sh

@@ -27,10 +27,10 @@ help() {
 Usage: $0 [-h] [options]
    Description:
         This script gathers a number of metrics for use in the
-	report generation script. Which tests are run can be
-	configured on the commandline. Specifically enabling
-	individual tests will disable the 'all' option, unless
-	'all' is also specified last.
+        report generation script. Which tests are run can be
+        configured on the commandline. Specifically enabling
+        individual tests will disable the 'all' option, unless
+        'all' is also specified last.
    Options:
         -a,         Run all tests (default).
         -d,         Run the density tests.
@@ -50,7 +50,7 @@ init() {
 	local OPTIND
 	while getopts "adhnst" opt;do
 		case ${opt} in
-		a)	# all
+		a)
 		    RUN_ALL=1
 		    ;;
 		d)
@@ -87,10 +87,10 @@ run_density_ksm() {
 	echo "Running KSM density tests"
 
 	# Run the memory footprint test - the main test that
-	# KSM affects. Run for 20 containers (that gives us a fair
-	# view of how memory gets shared across containers), and
-	# a default timeout of 300s - if KSM has not settled down
-	# by then, just take the measurement.
+	# KSM affects. Run for a sufficient number of containers
+	# (that gives us a fair view of how memory gets shared across
+	# containers), and a large enough timeout  for KSM to settle.
+	# If KSM has not settled down by then, just take the measurement.
 	# 'auto' mode should detect when KSM has settled automatically.
 	bash density/docker_memory_usage.sh 20 300 auto
 
@@ -105,13 +105,15 @@ run_density_ksm() {
 	bash density/footprint_data.sh
 
 	# mysql - medium sized container
+	# Need to wait for mysql to boot and settle before we measure
 	export PAYLOAD_SLEEP="10"
 	export PAYLOAD="mysql"
 	PAYLOAD_ARGS=" --innodb_use_native_aio=0 --disable-log-bin"
 	PAYLOAD_RUNTIME_ARGS=" -m 4G -e MYSQL_ALLOW_EMPTY_PASSWORD=1"
 	bash density/footprint_data.sh
 
 	# elasticsearch - large container
+	# Need to wait for elasticsearch to boot and settle before we measure
 	export PAYLOAD_SLEEP="10"
 	export PAYLOAD="elasticsearch"
 	PAYLOAD_ARGS=" "
@@ -123,7 +125,8 @@ run_density() {
 	echo "Running non-KSM density tests"
 
 	# Run the density tests - no KSM, so no need to wait for settle
-	# (so set a token 5s wait). Take the measure across 20 containers.
+	# Set a token short timeout, and use enough containers to get a
+	# good average measurement.
 	bash density/docker_memory_usage.sh 20 5
 }
 
@@ -137,7 +140,11 @@ run_time() {
 }
 
 run_storage() {
-	echo "No storage tests to run"
+	echo "Running storage tests"
+
+	# Enable this if you want to test a volume mount
+	#export TEST_VOLUME_MOUNT=1
+	bash storage/fio.sh
 }
 
 run_network() {

metrics/report/report_dockerfile/fio-reads.R

@@ -0,0 +1,185 @@
+#!/usr/bin/env Rscript
+# Copyright (c) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Display details for `fio` random read storage IO tests.
+
+
+library(ggplot2)	# ability to plot nicely
+library(gridExtra)	# So we can plot multiple graphs together
+suppressMessages(suppressWarnings(library(ggpubr)))	# for ggtexttable
+suppressMessages(library(jsonlite))			# to load the data
+suppressMessages(suppressWarnings(library(tidyr)))	# for gather
+library(tibble)
+
+resultsfiles=c(
+	"fio-randread-128.json",
+	"fio-randread-256.json",
+	"fio-randread-512.json",
+	"fio-randread-1k.json",
+	"fio-randread-2k.json",
+	"fio-randread-4k.json",
+	"fio-randread-8k.json",
+	"fio-randread-16k.json",
+	"fio-randread-32k.json",
+	"fio-randread-64k.json"
+	)
+
+data2=c()
+all_ldata=c()
+all_ldata2=c()
+stats=c()
+rstats=c()
+rstats_names=c()
+
+# For each set of results
+for (currentdir in resultdirs) {
+	dirstats=c()
+	for (resultsfile in resultsfiles) {
+		fname=paste(inputdir, currentdir, resultsfile, sep="")
+		if ( !file.exists(fname)) {
+			#warning(paste("Skipping non-existent file: ", fname))
+			next
+		}
+
+		# Derive the name from the test result dirname
+		datasetname=basename(currentdir)
+
+		# Import the data
+		fdata=fromJSON(fname)
+
+		blocksize=fdata$Raw$'global options'$bs
+
+		# Extract the latency data - it comes as a table of percentiles, so
+		# we have to do a little work...
+		clat=data.frame(clat_ns=fdata$Raw$jobs[[1]]$read$clat_ns$percentile)
+
+		# Generate a clat data set with 'clean' percentile numbers so
+		# we can sensibly plot it later on.
+		clat2=clat
+		colnames(clat2)<-sub("clat_ns.", "", colnames(clat2))
+		colnames(clat2)<-sub("0000", "", colnames(clat2))
+		ldata2=gather(clat2)
+		colnames(ldata2)[colnames(ldata2)=="key"] <- "percentile"
+		colnames(ldata2)[colnames(ldata2)=="value"] <- "ms"
+		ldata2$ms=ldata2$ms/1000000	#ns->ms
+		ldata2=cbind(ldata2, runtime=rep(datasetname, length(ldata2$percentile)))
+		ldata2=cbind(ldata2, blocksize=rep(blocksize, length(ldata2$percentile)))
+
+		# Pull the 95 and 99 percentile numbers for the boxplot
+		# Plotting all values for all runtimes and blocksizes is just way too
+		# noisy to make a meaninful picture, so we use this subset.
+		# Our values fall more in the range of ms...
+		pc95data=tibble(percentile=clat$clat_ns.95.000000/1000000)
+		pc95data=cbind(pc95data, runtime=rep(paste(datasetname, "95pc", sep="-"), length(pc95data$percentile)))
+		pc99data=tibble(percentile=clat$clat_ns.99.000000/1000000)
+		pc99data=cbind(pc99data, runtime=rep(paste(datasetname, "99pc", sep="-"), length(pc95data$percentile)))
+		ldata=rbind(pc95data, pc99data)
+		ldata=cbind(ldata, blocksize=rep(blocksize, length(ldata$percentile)))
+
+		# We want total bandwidth, so that is the sum of the bandwidths
+		# from all the read 'jobs'.
+		mdata=data.frame(read_bw_mps=as.numeric(sum(fdata$Raw$jobs[[1]]$read$bw)/1024))
+		mdata=cbind(mdata, iops_tot=as.numeric(sum(fdata$Raw$jobs[[1]]$read$iops)))
+		mdata=cbind(mdata, runtime=rep(datasetname, length(mdata[, "read_bw_mps"]) ))
+		mdata=cbind(mdata, blocksize=rep(blocksize, length(mdata[, "read_bw_mps"]) ))
+
+		# Collect up as sets across all files and runtimes.
+		data2=rbind(data2, mdata)
+		all_ldata=rbind(all_ldata, ldata)
+		all_ldata2=rbind(all_ldata2, ldata2)
+	}
+}
+
+# Bandwidth line plot
+read_bw_line_plot <- ggplot() +
+	geom_line( data=data2, aes(blocksize, read_bw_mps, group=runtime, color=runtime)) +
+	ylim(0, NA) +
+	ggtitle("Random Read total bandwidth") +
+	xlab("Blocksize") +
+	ylab("Bandwidth (MiB/s)") +
+	theme(
+		axis.text.x=element_text(angle=90),
+		legend.position=c(0.35,0.8),
+		legend.title=element_text(size=5),
+		legend.text=element_text(size=5),
+		legend.background = element_rect(fill=alpha('blue', 0.2))
+	)
+
+# IOPS line plot
+read_iops_line_plot <- ggplot() +
+	geom_line( data=data2, aes(blocksize, iops_tot, group=runtime, color=runtime)) +
+	ylim(0, NA) +
+	ggtitle("Random Read total IOPS") +
+	xlab("Blocksize") +
+	ylab("IOPS") +
+	theme(
+		axis.text.x=element_text(angle=90),
+		legend.position=c(0.35,0.8),
+		legend.title=element_text(size=5),
+		legend.text=element_text(size=5),
+		legend.background = element_rect(fill=alpha('blue', 0.2))
+	)
+
+# 95 and 99 percentile box plot
+read_clat_box_plot <- ggplot() +
+	geom_boxplot( data=all_ldata, aes(blocksize, percentile, color=runtime)) +
+	ylim(0, NA) +
+	ggtitle("Random Read completion latency", subtitle="95&98 percentiles, boxplot over jobs") +
+	xlab("Blocksize") +
+	ylab("Latency (ms)") +
+	theme(axis.text.x=element_text(angle=90)) +
+	# Use the 'paired' colour matrix as we are setting these up as pairs of
+	# 95 and 99 percentiles, and it is much easier to visually group those to
+	# each runtime if we use this colourmap.
+	scale_colour_brewer(palette="Paired")
+#	it would be nice to use the same legend theme as the other plots on this
+#	page, but because of the number of entries it tends to flow off the picture.
+#	theme(
+#		axis.text.x=element_text(angle=90),
+#		legend.position=c(0.35,0.8),
+#		legend.title=element_text(size=5),
+#		legend.text=element_text(size=5),
+#		legend.background = element_rect(fill=alpha('blue', 0.2))
+#	)
+
+# As the boxplot is actually quite hard to interpret, also show a linegraph
+# of all the percentiles for a single blocksize.
+which_blocksize='4k'
+clat_line_subtitle=paste("For blocksize", which_blocksize, sep=" ")
+single_blocksize=subset(all_ldata2, blocksize==which_blocksize)
+clat_line=aggregate(
+	single_blocksize$ms,
+	by=list(
+		percentile=single_blocksize$percentile,
+		blocksize=single_blocksize$blocksize,
+		runtime=single_blocksize$runtime
+	),
+	FUN=mean
+)
+
+clat_line$percentile=as.numeric(clat_line$percentile)
+
+read_clat_line_plot <- ggplot() +
+	geom_line( data=clat_line, aes(percentile, x, group=runtime, color=runtime)) +
+	ylim(0, NA) +
+	ggtitle("Random Read completion latency percentiles", subtitle=clat_line_subtitle) +
+	xlab("Percentile") +
+	ylab("Time (ms)") +
+	theme(
+		axis.text.x=element_text(angle=90),
+		legend.position=c(0.35,0.8),
+		legend.title=element_text(size=5),
+		legend.text=element_text(size=5),
+		legend.background = element_rect(fill=alpha('blue', 0.2))
+	)
+
+master_plot = grid.arrange(
+	read_bw_line_plot,
+	read_iops_line_plot,
+	read_clat_box_plot,
+	read_clat_line_plot,
+	nrow=2,
+	ncol=2 )
+