Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

test: improve the WPT runner and rename test files #24826

Closed
wants to merge 3 commits into from
Closed
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
test: improve WPT runner name matching
This patch:

- Support wildcards(*) in WPT runner name matching (needed by e.g.
  encoding where all the tests requires i18n support in the build)
- Print failure reasons when encountering an expected failure
- Fix a bug in copyGlobalsFromObject (previously it copies
  properties from `global` instead of the given `obj`)

Previously an expected failure is printed as

```
[EXPECTED_FAILURE] response.formData() with input: %61+%4d%4D=
```

Now it is printed as

```
[EXPECTED_FAILURE] response.formData() with input: %61+%4d%4D=
missing Request and Response
```
joyeecheung committed Dec 4, 2018

Verified

This commit was signed with the committer’s verified signature.
joyeecheung Joyee Cheung
commit b7d41c1da33b2c192b0a0d703660a086491948e5
137 changes: 98 additions & 39 deletions test/common/wpt.js
Original file line number Diff line number Diff line change
@@ -66,20 +66,92 @@ class ResourceLoader {
}
}

class StatusRule {
constructor(key, value, pattern = undefined) {
this.key = key;
this.requires = value.requires || [];
this.fail = value.fail;
this.skip = value.skip;
if (pattern) {
this.pattern = this.transformPattern(pattern);
}
// TODO(joyeecheung): implement this
this.scope = value.scope;
this.comment = value.comment;
}

/**
* Transform a filename pattern into a RegExp
* @param {string} pattern
* @returns {RegExp}
*/
transformPattern(pattern) {
const result = pattern.replace(/[-/\\^$+?.()|[\]{}]/g, '\\$&');
return new RegExp(result.replace('*', '.*'));
}
}

class StatusRuleSet {
constructor() {
// We use two sets of rules to speed up matching
this.exactMatch = {};
this.patternMatch = [];
}

/**
* @param {object} rules
*/
addRules(rules) {
for (const key of Object.keys(rules)) {
if (key.includes('*')) {
this.patternMatch.push(new StatusRule(key, rules[key], key));
} else {
this.exactMatch[key] = new StatusRule(key, rules[key]);
}
}
}

match(file) {
const result = [];
const exact = this.exactMatch[file];
if (exact) {
result.push(exact);
}
for (const item of this.patternMatch) {
if (item.pattern.test(file)) {
result.push(item);
}
}
return result;
}
}

class WPTTest {
/**
* @param {string} mod
* @param {string} filename
* @param {string[]} requires
* @param {string | undefined} failReason
* @param {string | undefined} skipReason
* @param {StatusRule[]} rules
*/
constructor(mod, filename, requires, failReason, skipReason) {
constructor(mod, filename, rules) {
this.module = mod; // name of the WPT module, e.g. 'url'
this.filename = filename; // name of the test file
this.requires = requires;
this.failReason = failReason;
this.skipReason = skipReason;

this.requires = new Set();
this.failReasons = [];
this.skipReasons = [];
for (const item of rules) {
if (item.requires.length) {
for (const req of item.requires) {
this.requires.add(req);
}
}
if (item.fail) {
this.failReasons.push(item.fail);
}
if (item.skip) {
this.skipReasons.push(item.skip);
}
}
}

getAbsolutePath() {
@@ -90,54 +162,37 @@ class WPTTest {
return fs.readFileSync(this.getAbsolutePath(), 'utf8');
}

shouldSkip() {
return this.failReason || this.skipReason;
}

requireIntl() {
return this.requires.includes('intl');
return this.requires.has('intl');
}
}

class StatusLoader {
constructor(path) {
this.path = path;
this.loaded = false;
this.status = null;
this.rules = new StatusRuleSet();
/** @type {WPTTest[]} */
this.tests = [];
}

loadTest(file) {
let requires = [];
let failReason;
let skipReason;
if (this.status[file]) {
requires = this.status[file].requires || [];
failReason = this.status[file].fail;
skipReason = this.status[file].skip;
}
return new WPTTest(this.path, file, requires,
failReason, skipReason);
}

load() {
const dir = path.join(__dirname, '..', 'wpt');
const statusFile = path.join(dir, 'status', `${this.path}.json`);
const result = JSON.parse(fs.readFileSync(statusFile, 'utf8'));
this.status = result;
this.rules.addRules(result);

const list = fs.readdirSync(fixtures.path('wpt', this.path));

for (const file of list) {
this.tests.push(this.loadTest(file));
if (!(/\.\w+\.js$/.test(file))) {
continue;
}
const match = this.rules.match(file);
this.tests.push(new WPTTest(this.path, file, match));
}
this.loaded = true;
}

get jsTests() {
return this.tests.filter((test) => test.filename.endsWith('.js'));
}
}

const PASSED = 1;
@@ -156,7 +211,7 @@ class WPTRunner {
this.status = new StatusLoader(path);
this.status.load();
this.tests = new Map(
this.status.jsTests.map((item) => [item.filename, item])
this.status.tests.map((item) => [item.filename, item])
);

this.results = new Map();
@@ -171,7 +226,10 @@ class WPTRunner {
*/
copyGlobalsFromObject(obj, names) {
for (const name of names) {
const desc = Object.getOwnPropertyDescriptor(global, name);
const desc = Object.getOwnPropertyDescriptor(obj, name);
if (!desc) {
assert.fail(`${name} does not exist on the object`);
}
this.globals.set(name, desc);
}
}
@@ -328,8 +386,9 @@ class WPTRunner {
for (const item of items) {
switch (item.type) {
case FAILED: {
if (test.failReason) {
if (test.failReasons.length) {
console.log(`[EXPECTED_FAILURE] ${item.test.name}`);
console.log(test.failReasons.join('; '));
} else {
console.log(`[UNEXPECTED_FAILURE] ${item.test.name}`);
unexpectedFailures.push([title, filename, item]);
@@ -386,10 +445,10 @@ class WPTRunner {
});
}

skip(filename, reason) {
skip(filename, reasons) {
this.addResult(filename, {
type: SKIPPED,
reason
reason: reasons.join('; ')
});
}

@@ -435,8 +494,8 @@ class WPTRunner {
const queue = [];
for (const test of this.tests.values()) {
const filename = test.filename;
if (test.skipReason) {
this.skip(filename, test.skipReason);
if (test.skipReasons.length > 0) {
this.skip(filename, test.skipReasons);
continue;
}