feat(API): builds on demand (#243)

packages/api/server.js

@@ -21,6 +21,7 @@ const GetRepoHandler = require('./src/get-repo.js');
 const GetOrgHandler = require('./src/get-org.js');
 const GetTestHandler = require('./src/get-test.js');
 const GetExportHandler = require('./src/get-export.js');
+const GetBatchesHandler = require('./src/get-batches.js');
 const client = require('./src/firestore.js');
 const auth = require('./src/auth.js');
 const { InvalidParameterError } = require('./lib/errors');
@@ -103,6 +104,8 @@ const getTestHandler = new GetTestHandler(app, client);
 getTestHandler.listen();
 const getExportHandler = new GetExportHandler(app, client);
 getExportHandler.listen();
+const getBatchesHandler = new GetBatchesHandler(app, client);
+getBatchesHandler.listen();
 
 const port = process.env.PORT ? Number(process.env.PORT) : 3000;
 const host = '0.0.0.0';

packages/api/src/get-batches.js

@@ -0,0 +1,117 @@
+
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// class to receive POSTS with build information
+
+const firebaseEncode = require('../lib/firebase-encode');
+const { InvalidParameterError, handleError } = require('../lib/errors');
+const { isJson } = require('../util/validation');
+const FILTERS_ALLOWED = ['tag', 'ref', 'os', 'matrix'];
+const BatchService = require('../util/batches.js');
+const moment = require('moment');
+const { isAPositiveNumber } = require('../util/validation.js');
+
+class GetBatchesHandler {
+  constructor (app, client) {
+    this.app = app;
+    this.client = client;
+    this.batchService = new BatchService();
+  }
+
+  _extractFiltersInQuery (req, starterQuery) {
+    FILTERS_ALLOWED.forEach(filter => {
+      let filterValue = req.query[filter];
+      if (filterValue) {
+        if (filter === 'matrix' && !isJson(filterValue)) throw new InvalidParameterError('matrix parameter must be json');
+        filterValue = '' + filterValue; // sanitize the parameter
+        starterQuery = starterQuery.where('environment.' + firebaseEncode(filter), '==', filterValue);
+      }
+    });
+    return starterQuery;
+  }
+
+  _extractUtcOffset (req) {
+    const offset = parseInt(req.query.utcOffset);
+    return isNaN(offset) ? 0 : offset;
+  }
+
+  async _getBuilds (req, utcOffset) {
+    const repoid = firebaseEncode(req.params.orgname + '/' + req.params.reponame);
+    let starterQuery = this.client.collection(global.headCollection).doc(repoid).collection('builds');
+
+    const initDate = moment.utc().utcOffset(utcOffset).subtract(45, 'weeks').startOf('day').toDate();
+    starterQuery = starterQuery.where('timestamp', '>=', initDate);
+
+    starterQuery = this._extractFiltersInQuery(req, starterQuery);
+
+    const snapshot = await starterQuery.orderBy('timestamp', 'asc').get();
+    const builds = [];
+    snapshot.forEach(doc => builds.push(doc.data()));
+
+    return builds;
+  }
+
+  async _getDayBuilds (req, timestamp, utcOffset) {
+    const repoid = firebaseEncode(req.params.orgname + '/' + req.params.reponame);
+
+    let starterQuery = this.client.collection(global.headCollection).doc(repoid).collection('builds');
+
+    const reqDay = moment.unix(timestamp).utc().utcOffset(utcOffset, false);
+    const startOfDay = reqDay.startOf('day').toDate();
+    const endOfDay = reqDay.endOf('day').toDate();
+
+    starterQuery = starterQuery.where('timestamp', '>=', startOfDay);
+    starterQuery = starterQuery.where('timestamp', '<=', endOfDay);
+
+    starterQuery = this._extractFiltersInQuery(req, starterQuery);
+
+    const snapshot = await starterQuery.orderBy('timestamp', 'asc').get();
+    const builds = [];
+    snapshot.forEach(doc => builds.push(doc.data()));
+
+    return builds;
+  }
+
+  listen () {
+    // return all batches for a repository
+    this.app.get('/api/repo/:orgname/:reponame/batches', async (req, res) => {
+      try {
+        const utcOffset = this._extractUtcOffset(req);
+
+        const builds = await this._getBuilds(req, utcOffset);
+        const batches = this.batchService.buildBatches(builds, utcOffset);
+        res.send(batches);
+      } catch (err) {
+        handleError(res, err);
+      }
+    });
+
+    // return all builds for a day
+    this.app.get('/api/repo/:orgname/:reponame/batch/:timestamp', async (req, res) => {
+      try {
+        const timestamp = req.params.timestamp;
+        if (!isAPositiveNumber(timestamp)) throw new InvalidParameterError('timestamp parameter must be a positive number');
+        const utcOffset = this._extractUtcOffset(req);
+
+        const builds = await this._getDayBuilds(req, timestamp, utcOffset);
+        res.send(builds);
+      } catch (err) {
+        handleError(res, err);
+      }
+    });
+  }
+}
+
+module.exports = GetBatchesHandler;

packages/api/test/batch-service.js

@@ -0,0 +1,82 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+const { describe, beforeEach, it } = require('mocha');
+const assert = require('assert');
+const BatchService = require('../util/batches.js');
+const mockBuilds = require('./mockBuilds');
+
+describe('BatchService', () => {
+  let batchService;
+
+  describe('buildBatches', () => {
+    beforeEach(() => { batchService = new BatchService(); });
+
+    it('should return a batch corresponding to the provided build', () => {
+      const builds = [mockBuilds._3PreviousDays[0]];
+
+      const batches = batchService.buildBatches(builds);
+
+      assert.strictEqual(batches.length, 1);
+      assert.strictEqual(batches[0].passedBuilds, 1);
+      assert.strictEqual(batches[0].failingBuilds, 0);
+      assert.strictEqual(batches[0].flakyBuilds, 0);
+    });
+
+    it('should return batches for builds run on different days', () => {
+      const builds = [
+        mockBuilds._3PreviousDays[0],
+        mockBuilds._3PreviousDays[2],
+        mockBuilds._3PreviousDays[3]
+      ];
+
+      const batches = batchService.buildBatches(builds);
+
+      assert.strictEqual(batches.length, 3);
+    });
+
+    it('should return batches with the provided utc offset', () => {
+      const builds = mockBuilds._3PreviousDays;
+      const utcOffset = 1;
+
+      const batches = batchService.buildBatches(builds, utcOffset);
+
+      assert.strictEqual(batches.length, 4);
+      batches.forEach((batch, index) => assert.strictEqual(batch.timestamp, builds[index].timestamp._seconds));
+    });
+
+    it('should return batches with stats correponding to the contained builds', () => {
+      const builds = mockBuilds._3PreviousDays;
+
+      const batches = batchService.buildBatches(builds);
+
+      assert.strictEqual(batches.length, 3);
+
+      // 2 passing builds
+      assert.strictEqual(batches[0].passedBuilds, 2);
+      assert.strictEqual(batches[0].failingBuilds, 0);
+      assert.strictEqual(batches[0].flakyBuilds, 0);
+
+      // 1 failing build
+      assert.strictEqual(batches[1].passedBuilds, 0);
+      assert.strictEqual(batches[1].failingBuilds, 1);
+      assert.strictEqual(batches[1].flakyBuilds, 0);
+
+      // 1 flaky and passing build
+      assert.strictEqual(batches[2].passedBuilds, 1);
+      assert.strictEqual(batches[2].failingBuilds, 0);
+      assert.strictEqual(batches[2].flakyBuilds, 1);
+    });
+  });
+});