-
Notifications
You must be signed in to change notification settings - Fork 0
/
run-tests.js
135 lines (117 loc) · 4.39 KB
/
run-tests.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
require("dotenv").config();
const path = require("path");
const fs = require("fs").promises;
const shortHash = require("short-hash");
const fetch = require("node-fetch");
const fastglob = require("fast-glob");
const PerfLeaderboard = require("performance-leaderboard");
const NUMBER_OF_RUNS = 3;
const FREQUENCY = 60; // in minutes
const NETLIFY_MAX_LIMIT = 15; // in minutes, netlify limit
const ESTIMATED_MAX_TIME_PER_TEST = 0.75; // in minutes, estimate based on looking at past builds
const prettyTime = (seconds) => {
// Based on https://johnresig.com/blog/javascript-pretty-date/
const days = Math.floor(seconds / (60*60*24));
return (
(days === 0 &&
((seconds < 60 && "just now") ||
(seconds < 60 * 2 && "1 minute ago") ||
(seconds < 3600 && Math.floor(seconds / 60) + " minutes ago") ||
(seconds < 7200 && "1 hour ago") ||
(seconds < 86400 * 2 && Math.floor(seconds / 3600) + " hours ago"))) ||
(days < 7 && days + " days ago") ||
(Math.ceil(days / 7) + " weeks ago")
);
}
async function tryToPreventNetlifyBuildTimeout(dateTestsStarted, numberOfUrls) {
let minutesRemaining = NETLIFY_MAX_LIMIT - (Date.now() - dateTestsStarted)/(1000*60)
if(process.env.CONTEXT &&
process.env.CONTEXT === "production" &&
NETLIFY_MAX_LIMIT &&
minutesRemaining < numberOfUrls * ESTIMATED_MAX_TIME_PER_TEST) {
console.log( `run-tests has about ${minutesRemaining} minutes left, but the next run has ${numberOfUrls} urls. Saving it for the next build.` );
return true;
}
return false;
}
(async function() {
// Netlify specific check (works fine without this env variable too)
if(process.env.CONTEXT && process.env.CONTEXT !== "production") {
console.log( "Skipping all test runs because we’re in a Netlify build or deploy preview!" );
return;
}
let dateTestsStarted = Date.now();
let dataDir = `./_data/`;
// Careful here, this filename needs to be .gitignore’d and
// listed in the keep-data-cache plugin.
let lastRunsFilename = `${dataDir}results-last-runs.json`;
let lastRuns;
try {
lastRuns = require(lastRunsFilename);
console.log( "Last runs at start: ", JSON.stringify(lastRuns) );
} catch (e) {
console.log(`There are no known last run timestamps`);
lastRuns = {};
}
let verticals = await fastglob("./_data/sites/*.js", {
caseSensitiveMatch: false
});
for(let file of verticals) {
let group = require(file);
if(typeof group === "function") {
group = await group();
}
let key = file.split("/").pop().replace(/\.js$/, "");
if(group.skip) {
console.log( `Skipping ${key} (you told me to in your site config)` );
continue;
}
if(await tryToPreventNetlifyBuildTimeout(dateTestsStarted, group.urls.length)) {
// stop everything, we’re too close to the timeout
return;
}
let runFrequency =
group.options && group.options.frequency
? group.options.frequency
: FREQUENCY;
if (!lastRuns[key]) {
console.log(`First tests for ${key}.`);
} else {
const lastRun = lastRuns[key];
const lastRunSecondsAgo = (dateTestsStarted - lastRun.timestamp) / 1000;
const lastRunSecondsAgoPretty = prettyTime(lastRunSecondsAgo);
const lastRunMinutesAgo = lastRunSecondsAgo / 60;
if (lastRunMinutesAgo < runFrequency) {
console.log(
`Previous test for ${key} ran ${lastRunSecondsAgoPretty}, less than ${runFrequency} minutes, skipping.`
);
continue;
} else {
console.log(`Previous test for ${key} ran ${lastRunSecondsAgoPretty}, more than ${runFrequency} minutes, running.`);
}
}
let runCount =
group.options && group.options.runs ? group.options.runs : NUMBER_OF_RUNS;
let results = await PerfLeaderboard(
group.urls,
runCount,
group.options || {}
);
let promises = [];
for(let result of results) {
let id = shortHash(result.url);
let isIsolated = group.options && group.options.isolated;
let dir = `${dataDir}results/${isIsolated ? `${key}/` : ""}${id}/`;
let filename = `${dir}date-${dateTestsStarted}.json`;
await fs.mkdir(dir, { recursive: true });
promises.push(fs.writeFile(filename, JSON.stringify(result, null, 2)));
console.log( `Writing ${filename}.` );
}
await Promise.all(promises);
lastRuns[key] = { timestamp: Date.now() };
console.log( `Finished testing "${key}".` );
// Write the last run time to avoid re-runs
await fs.writeFile(lastRunsFilename, JSON.stringify(lastRuns, null, 2));
console.log( `Last runs after "${key}":`, JSON.stringify(lastRuns) );
}
})();