Validate threshold configuration before starting the execution

This commit makes sure that the threshold configuration (as passed in
the script exported options for instance) is valid, before starting the
execution.

A valid threshold must pass the following assertion:
- Its expression is syntaxically correct and is parsable
- It applies to a metrics that's known to k6, either builtin or custom
- Its expression's aggregation method is valid for the metric it applies
to

Threshold validation will be made in the context of the `run`, `cloud`,
`archive`, `inspect`, and `archive` commands. If a threshold definition
is invalid, the k6 program will exit with
a status code of 104.

ref #2330

cmd/archive.go

@@ -75,7 +75,7 @@ An archive is a fully self-contained test run, and can be executed identically e
 				return err
 			}
 
-			_, err = deriveAndValidateConfig(conf, r.IsExecutable, logger)
+			_, err = deriveAndValidateConfig(conf, registry, r.IsExecutable, logger)
 			if err != nil {
 				return err
 			}

cmd/cloud.go

@@ -114,7 +114,7 @@ This will execute the test on the k6 cloud service. Use "k6 login cloud" to auth
 				return err
 			}
 
-			derivedConf, err := deriveAndValidateConfig(conf, r.IsExecutable, logger)
+			derivedConf, err := deriveAndValidateConfig(conf, registry, r.IsExecutable, logger)
 			if err != nil {
 				return err
 			}

cmd/config_test.go

@@ -31,7 +31,6 @@ import (
 	"go.k6.io/k6/lib"
 	"go.k6.io/k6/lib/executor"
 	"go.k6.io/k6/lib/metrics"
-	"go.k6.io/k6/lib/testutils"
 	"go.k6.io/k6/lib/types"
 	"go.k6.io/k6/stats"
 	"gopkg.in/guregu/null.v3"

cmd/inspect.go

@@ -129,7 +129,7 @@ func addExecRequirements(b *js.Bundle,
 		return nil, err
 	}
 
-	conf, err = deriveAndValidateConfig(conf, runner.IsExecutable, logger)
+	conf, err = deriveAndValidateConfig(conf, registry, runner.IsExecutable, logger)
 	if err != nil {
 		return nil, err
 	}

cmd/run.go

@@ -51,6 +51,7 @@ import (
 	"go.k6.io/k6/lib/consts"
 	"go.k6.io/k6/lib/metrics"
 	"go.k6.io/k6/loader"
+	"go.k6.io/k6/stats"
 	"go.k6.io/k6/ui/pb"
 )
 
@@ -110,6 +111,10 @@ a commandline interface for interacting with it.`,
 			builtinMetrics := metrics.RegisterBuiltinMetrics(registry)
 			initRunner, err := newRunner(logger, src, globalFlags.runType, filesystems, runtimeOptions, builtinMetrics, registry)
 			if err != nil {
+				if errors.Is(err, stats.ErrThresholdParsing) {
+					return errext.WithExitCodeIfNone(err, exitcodes.InvalidConfig)
+				}
+
 				return common.UnwrapGojaInterruptedError(err)
 			}
 
@@ -125,7 +130,7 @@ a commandline interface for interacting with it.`,
 				return err
 			}
 
-			conf, err = deriveAndValidateConfig(conf, initRunner.IsExecutable, logger)
+			conf, err = deriveAndValidateConfig(conf, registry, initRunner.IsExecutable, logger)
 			if err != nil {
 				return err
 			}

cmd/testdata/thresholds/malformed_expression.js

@@ -0,0 +1,11 @@
+export const options = {
+	thresholds: {
+		http_reqs: ["foo>0"], // Counter
+	},
+};
+
+export default function () {
+	console.log(
+		"asserting that a malformed threshold fails with exit code 104 (Invalid config)"
+	);
+}

cmd/testdata/thresholds/non_existing_metric.js

@@ -0,0 +1,13 @@
+export const options = {
+	thresholds: {
+		// non existing is neither registered, nor a builtin metric.
+		// k6 should catch that.
+		"non existing": ["rate>0"],
+	},
+};
+
+export default function () {
+	console.log(
+		"asserting that a threshold over a non-existing metric fails with exit code 104 (Invalid config)"
+	);
+}

cmd/testdata/thresholds/unsupported_aggregation_method.js

@@ -0,0 +1,15 @@
+export const options = {
+	thresholds: {
+		// http_reqs is a Counter metric. As such, it supports
+		// only the 'count' and 'rate' operations. Thus, 'value'
+		// being a Gauge's metric aggregation method, the threshold
+		// configuration evaluation should fail.
+		http_reqs: ["value>0"],
+	},
+};
+
+export default function () {
+	console.log(
+		"asserting that a threshold applying a method over a metric not supporting it fails with exit code 104 (Invalid config)"
+	);
+}