From c2816efde4c5d8021ee36034dd61655a71bfa3f3 Mon Sep 17 00:00:00 2001 From: Paul Gschwendtner Date: Sat, 10 Jun 2017 01:09:26 +0200 Subject: [PATCH] build: payload github status (#4549) * build: payload github status * Show payload deta on PRs using the Github Statuses API. * Update config variables --- .../functions/github/github-status.ts | 2 +- tools/dashboard/functions/jwt/verify-token.ts | 4 +- tools/gulp/tasks/payload.ts | 117 +++++++++++++++--- tools/gulp/util/firebase.ts | 14 ++- tools/gulp/util/travis-ci.ts | 4 + 5 files changed, 118 insertions(+), 23 deletions(-) diff --git a/tools/dashboard/functions/github/github-status.ts b/tools/dashboard/functions/github/github-status.ts index c964df091431..4ffede05b9dc 100644 --- a/tools/dashboard/functions/github/github-status.ts +++ b/tools/dashboard/functions/github/github-status.ts @@ -4,7 +4,7 @@ const request = require('request'); const {version, name} = require('../package.json'); /** API token for the Github repository. Required to set the github status on commits and PRs. */ -const repoToken = config().repoToken; +const repoToken = config().secret.github; /** Data that must be specified to set a Github PR status. */ export type GithubStatusData = { diff --git a/tools/dashboard/functions/jwt/verify-token.ts b/tools/dashboard/functions/jwt/verify-token.ts index 6630fc030632..da987f31fbd6 100644 --- a/tools/dashboard/functions/jwt/verify-token.ts +++ b/tools/dashboard/functions/jwt/verify-token.ts @@ -2,10 +2,10 @@ import {verify} from 'jsonwebtoken'; import {config} from 'firebase-functions'; /** The JWT secret. This is used to validate JWT. */ -const jwtSecret = config().jwtSecret; +const jwtSecret = config().secret.jwt; /** The repo slug. This is used to validate the JWT is sent from correct repo. */ -const repoSlug = config().repoSlug; +const repoSlug = config().repo.slug; export function verifyToken(token: string): boolean { try { diff --git a/tools/gulp/tasks/payload.ts b/tools/gulp/tasks/payload.ts index f3c4e98f72db..0f12e90c0073 100644 --- a/tools/gulp/tasks/payload.ts +++ b/tools/gulp/tasks/payload.ts @@ -1,18 +1,20 @@ import {task} from 'gulp'; import {join} from 'path'; import {statSync} from 'fs'; -import {spawnSync} from 'child_process'; -import {isTravisMasterBuild} from '../util/travis-ci'; -import {openFirebaseDashboardApp} from '../util/firebase'; +import {isTravisBuild, isTravisMasterBuild} from '../util/travis-ci'; import {buildConfig} from '../packaging/build-config'; +import {openFirebaseDashboardApp, openFirebaseDashboardAppAsGuest} from '../util/firebase'; + +// These imports lack of type definitions. +const request = require('request'); /** Path to the directory where all bundles are living. */ const bundlesDir = join(buildConfig.outputDir, 'bundles'); /** Task which runs test against the size of material. */ -task('payload', ['material:clean-build'], () => { +task('payload', ['material:clean-build'], async () => { - let results = { + const results = { timestamp: Date.now(), // Material bundles material_umd: getBundleSize('material.umd.js'), @@ -29,9 +31,23 @@ task('payload', ['material:clean-build'], () => { // Print the results to the console, so we can read it from the CI. console.log('Payload Results:', JSON.stringify(results, null, 2)); - // Publish the results to firebase when it runs on Travis and not as a PR. - if (isTravisMasterBuild()) { - return publishResults(results); + if (isTravisBuild()) { + // Open a connection to Firebase. For PRs the connection will be established as a guest. + const firebaseApp = isTravisMasterBuild() ? + openFirebaseDashboardApp() : + openFirebaseDashboardAppAsGuest(); + const database = firebaseApp.database(); + const currentSha = process.env['TRAVIS_PULL_REQUEST_SHA'] || process.env['TRAVIS_COMMIT']; + + // Upload the payload results and calculate the payload diff in parallel. Otherwise the + // payload task will take much more time inside of Travis builds. + await Promise.all([ + uploadPayloadResults(database, currentSha, results), + calculatePayloadDiff(database, currentSha, results) + ]); + + // Disconnect database connection because Firebase otherwise prevents Gulp from exiting. + firebaseApp.delete(); } }); @@ -46,14 +62,81 @@ function getFilesize(filePath: string) { return statSync(filePath).size / 1000; } -/** Publishes the given results to the firebase database. */ -function publishResults(results: any) { - const latestSha = spawnSync('git', ['rev-parse', 'HEAD']).stdout.toString().trim(); - const dashboardApp = openFirebaseDashboardApp(); - const database = dashboardApp.database(); +/** + * Calculates the difference between the last and current library payload. + * The results will be published as a commit status on Github. + */ +async function calculatePayloadDiff(database: any, currentSha: string, currentPayload: any) { + const authToken = process.env['FIREBASE_ACCESS_TOKEN']; + + if (!authToken) { + console.error('Cannot calculate Payload diff because there is no "FIREBASE_ACCESS_TOKEN" ' + + 'environment variable set.'); + return; + } + + const previousPayload = await getLastPayloadResults(database); + + if (!previousPayload) { + console.warn('There are no previous payload results uploaded. Cannot calculate payload ' + + 'difference for this job'); + return; + } + + // Calculate library sizes by combining the CDK and Material FESM 2015 bundles. + const previousSize = previousPayload.cdk_fesm_2015 + previousPayload.material_fesm_2015; + const currentSize = currentPayload.cdk_fesm_2015 + currentPayload.material_fesm_2015; + const deltaSize = currentSize - previousSize; + + // Update the Github status of the current commit by sending a request to the dashboard + // firebase http trigger function. + await updateGithubStatus(currentSha, deltaSize, authToken); +} + +/** + * Updates the Github status of a given commit by sending a request to a Firebase function of + * the dashboard. The function has access to the Github repository and can set status for PRs too. + */ +async function updateGithubStatus(commitSha: string, payloadDiff: number, authToken: string) { + const options = { + url: 'https://us-central1-material2-board.cloudfunctions.net/payloadGithubStatus', + headers: { + 'User-Agent': 'Material2/PayloadTask', + 'auth-token': authToken, + 'commit-sha': commitSha, + 'commit-payload-diff': payloadDiff + } + }; + + return new Promise((resolve, reject) => { + request(options, (err: any, response: any, body: string) => { + if (err) { + reject(`Dashboard Error ${err.toString()}`); + } else { + console.info('Dashboard Response:', JSON.parse(body).message); + resolve(response.statusCode); + } + }); + }); +} + +/** Uploads the current payload results to the Dashboard database. */ +async function uploadPayloadResults(database: any, currentSha: string, currentPayload: any) { + if (isTravisMasterBuild()) { + await database.ref('payloads').child(currentSha).set(currentPayload); + } +} + +/** Gets the last payload uploaded from previous Travis builds. */ +async function getLastPayloadResults(database: admin.database.Database) { + const snapshot = await database.ref('payloads') + .orderByChild('timestamp') + .limitToLast(1) + .once('value'); + + // The value of the DataSnapshot is an object with the SHA as a key. Later only the + // first value of the object will be returned because the SHA is unnecessary. + const results = snapshot.val(); - // Write the results to the payloads object with the latest Git SHA as key. - return database.ref('payloads').child(latestSha).set(results) - .catch((err: any) => console.error(err)) - .then(() => dashboardApp.delete()); + return snapshot.hasChildren() ? results[Object.keys(results)[0]] : null; } diff --git a/tools/gulp/util/firebase.ts b/tools/gulp/util/firebase.ts index 7e852907e753..d185ab978b59 100644 --- a/tools/gulp/util/firebase.ts +++ b/tools/gulp/util/firebase.ts @@ -5,11 +5,15 @@ const cloudStorage = require('@google-cloud/storage'); // Firebase configuration for the Screenshot project. Use the config from the screenshot functions. const screenshotFirebaseConfig = require('../../screenshot-test/functions/config.json'); -/** Opens a connection to the Firebase dashboard app. */ -export function openFirebaseDashboardApp() { +/** Database URL of the dashboard firebase project.*/ +const dashboardDatabaseUrl = 'https://material2-board.firebaseio.com'; + +/** Opens a connection to the Firebase dashboard app using a service account. */ +export function openFirebaseDashboardApp(asGuest = false) { // Initialize the Firebase application with firebaseAdmin credentials. // Credentials need to be for a Service Account, which can be created in the Firebase console. return firebaseAdmin.initializeApp({ + databaseURL: dashboardDatabaseUrl, credential: firebaseAdmin.credential.cert({ project_id: 'material2-board', client_email: 'material2-board@appspot.gserviceaccount.com', @@ -17,10 +21,14 @@ export function openFirebaseDashboardApp() { // The line-breaks need to persist in the service account private key. private_key: decode(process.env['MATERIAL2_BOARD_FIREBASE_SERVICE_KEY']) }), - databaseURL: 'https://material2-board.firebaseio.com' }); } +/** Opens a connection to the Firebase dashboard app with no authentication. */ +export function openFirebaseDashboardAppAsGuest() { + return firebase.initializeApp({ databaseURL: dashboardDatabaseUrl }); +} + /** * Open Google Cloud Storage for screenshots. * The files uploaded to google cloud are also available to firebase storage. diff --git a/tools/gulp/util/travis-ci.ts b/tools/gulp/util/travis-ci.ts index bd8e328b29c8..7267fd99029b 100644 --- a/tools/gulp/util/travis-ci.ts +++ b/tools/gulp/util/travis-ci.ts @@ -2,3 +2,7 @@ export function isTravisMasterBuild() { return process.env['TRAVIS_PULL_REQUEST'] === 'false'; } + +export function isTravisBuild() { + return process.env['TRAVIS'] === 'true'; +}