Breaking: Validate test options (refs eslint/eslint#2179)

lib/eslint-tester.js

@@ -48,7 +48,10 @@ var assert = require("chai").assert,
     path = require("path"),
     merge = require("lodash.merge"),
     omit = require("lodash.omit"),
-    clone = require("lodash.clonedeep");
+    clone = require("lodash.clonedeep"),
+    validator = require("eslint").validator,
+    validate = require("is-my-json-valid"),
+    metaSchema = require("./json-schema-schema.json");
 
 //------------------------------------------------------------------------------
 // Private Members
@@ -67,6 +70,8 @@ var eslintTesterParameters = [
     "errors"
 ];
 
+var validateSchema = validate(metaSchema, { verbose: true });
+
 //------------------------------------------------------------------------------
 // Public Interface
 //------------------------------------------------------------------------------
@@ -145,8 +150,7 @@ ESLintTester.prototype = {
 
         function runRuleForItem(ruleName, item) {
             var config = clone(testerConfig),
-                code,
-                filename;
+                code, filename, schema;
 
             if (typeof item === "string") {
                 code = item;
@@ -175,8 +179,20 @@ ESLintTester.prototype = {
                 config.rules[ruleName] = item.args ? item.args : 1;
             }
 
-
             eslint.defineRule(ruleName, require(path.resolve(process.cwd(), rulePath)));
+
+            schema = validator.getRuleOptionsSchema(ruleName);
+            validateSchema(schema);
+            if (validateSchema.errors) {
+                throw new Error([
+                    "Schema for rule " + ruleName + " is invalid:"
+                ].concat(validateSchema.errors.map(function (error) {
+                    return "\t" + error.field + ": " + error.message;
+                })).join("\n"));
+            }
+
+            validator.validate(config, "eslint-tester");
+
             return eslint.verify(code, config, filename);
         }
 

lib/json-schema-schema.json

@@ -0,0 +1,150 @@
+{
+    "id": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "description": "Core schema meta-schema",
+    "definitions": {
+        "schemaArray": {
+            "type": "array",
+            "minItems": 1,
+            "items": { "$ref": "#" }
+        },
+        "positiveInteger": {
+            "type": "integer",
+            "minimum": 0
+        },
+        "positiveIntegerDefault0": {
+            "allOf": [ { "$ref": "#/definitions/positiveInteger" }, { "default": 0 } ]
+        },
+        "simpleTypes": {
+            "enum": [ "array", "boolean", "integer", "null", "number", "object", "string" ]
+        },
+        "stringArray": {
+            "type": "array",
+            "items": { "type": "string" },
+            "minItems": 1,
+            "uniqueItems": true
+        }
+    },
+    "type": "object",
+    "properties": {
+        "id": {
+            "type": "string",
+            "format": "uri"
+        },
+        "$schema": {
+            "type": "string",
+            "format": "uri"
+        },
+        "title": {
+            "type": "string"
+        },
+        "description": {
+            "type": "string"
+        },
+        "default": {},
+        "multipleOf": {
+            "type": "number",
+            "minimum": 0,
+            "exclusiveMinimum": true
+        },
+        "maximum": {
+            "type": "number"
+        },
+        "exclusiveMaximum": {
+            "type": "boolean",
+            "default": false
+        },
+        "minimum": {
+            "type": "number"
+        },
+        "exclusiveMinimum": {
+            "type": "boolean",
+            "default": false
+        },
+        "maxLength": { "$ref": "#/definitions/positiveInteger" },
+        "minLength": { "$ref": "#/definitions/positiveIntegerDefault0" },
+        "pattern": {
+            "type": "string",
+            "format": "regex"
+        },
+        "additionalItems": {
+            "anyOf": [
+                { "type": "boolean" },
+                { "$ref": "#" }
+            ],
+            "default": {}
+        },
+        "items": {
+            "anyOf": [
+                { "$ref": "#" },
+                { "$ref": "#/definitions/schemaArray" }
+            ],
+            "default": {}
+        },
+        "maxItems": { "$ref": "#/definitions/positiveInteger" },
+        "minItems": { "$ref": "#/definitions/positiveIntegerDefault0" },
+        "uniqueItems": {
+            "type": "boolean",
+            "default": false
+        },
+        "maxProperties": { "$ref": "#/definitions/positiveInteger" },
+        "minProperties": { "$ref": "#/definitions/positiveIntegerDefault0" },
+        "required": { "$ref": "#/definitions/stringArray" },
+        "additionalProperties": {
+            "anyOf": [
+                { "type": "boolean" },
+                { "$ref": "#" }
+            ],
+            "default": {}
+        },
+        "definitions": {
+            "type": "object",
+            "additionalProperties": { "$ref": "#" },
+            "default": {}
+        },
+        "properties": {
+            "type": "object",
+            "additionalProperties": { "$ref": "#" },
+            "default": {}
+        },
+        "patternProperties": {
+            "type": "object",
+            "additionalProperties": { "$ref": "#" },
+            "default": {}
+        },
+        "dependencies": {
+            "type": "object",
+            "additionalProperties": {
+                "anyOf": [
+                    { "$ref": "#" },
+                    { "$ref": "#/definitions/stringArray" }
+                ]
+            }
+        },
+        "enum": {
+            "type": "array",
+            "minItems": 1,
+            "uniqueItems": true
+        },
+        "type": {
+            "anyOf": [
+                { "$ref": "#/definitions/simpleTypes" },
+                {
+                    "type": "array",
+                    "items": { "$ref": "#/definitions/simpleTypes" },
+                    "minItems": 1,
+                    "uniqueItems": true
+                }
+            ]
+        },
+        "allOf": { "$ref": "#/definitions/schemaArray" },
+        "anyOf": { "$ref": "#/definitions/schemaArray" },
+        "oneOf": { "$ref": "#/definitions/schemaArray" },
+        "not": { "$ref": "#" }
+    },
+    "dependencies": {
+        "exclusiveMaximum": [ "maximum" ],
+        "exclusiveMinimum": [ "minimum" ]
+    },
+    "default": {}
+}

package.json

@@ -28,6 +28,7 @@
     "dateformat": "^1.0.11",
     "eslint": "latest",
     "esprima": "^2.1.0",
+    "is-my-json-valid": "^2.10.1",
     "istanbul": "^0.3.5",
     "rewire": "^2.1.3",
     "shelljs": "^0.3.0",

tests/fixtures/no-invalid-schema.js

@@ -0,0 +1,28 @@
+/**
+ ** @fileoverview Test rule to flag invalid schemas
+ ** @author Brandon Mills
+ **/
+
+//------------------------------------------------------------------------------
+// Rule Definition
+//------------------------------------------------------------------------------
+
+module.exports = function(context) {
+    "use strict";
+
+    var config = context.options[0];
+
+    return {
+        "Program": function(node) {
+            if (config) {
+                context.report(node, "Expected nothing.");
+            }
+        }
+    };
+};
+
+module.exports.schema = [
+    {
+        "enum": []
+    }
+];

tests/fixtures/no-schema-violation.js

@@ -0,0 +1,28 @@
+/**
+ ** @fileoverview Test rule to flag schema violations
+ ** @author Brandon Mills
+ **/
+
+//------------------------------------------------------------------------------
+// Rule Definition
+//------------------------------------------------------------------------------
+
+module.exports = function(context) {
+    "use strict";
+
+    var config = context.options[0];
+
+    return {
+        "Program": function(node) {
+            if (config && config !== "foo") {
+                context.report(node, "Expected foo.");
+            }
+        }
+    };
+};
+
+module.exports.schema = [
+    {
+        "enum": ["foo"]
+    }
+];

tests/lib/eslint-tester.js

@@ -353,6 +353,38 @@ describe("ESLintTester", function() {
         });
     });
 
+    it("should prevent invalid options schemas", function() {
+
+        assert.throws(function() {
+            eslintTester.addRuleTest("tests/fixtures/no-invalid-schema", {
+                valid: [
+                    "var answer = 6 * 7;",
+                    { code: "var answer = 6 * 7;", options: [] }
+                ],
+                invalid: [
+                    { code: "var answer = 6 * 7;", options: ["bar"], errors: [{ message: "Expected nothing." }] }
+                ]
+            });
+        }, /Schema for rule .* is invalid/);
+
+    });
+
+    it("should prevent schema violations in options", function() {
+
+        assert.throws(function() {
+            eslintTester.addRuleTest("tests/fixtures/no-schema-violation", {
+                valid: [
+                    "var answer = 6 * 7;",
+                    { code: "var answer = 6 * 7;", options: ["foo"] }
+                ],
+                invalid: [
+                    { code: "var answer = 6 * 7;", options: ["bar"], errors: [{ message: "Expected foo." }] }
+                ]
+            });
+        }, /Value "bar" must be an enum value./);
+
+    });
+
 
 
     it("should throw an error if there are no valid tests", function() {