From eae1b7d9ee3269219a5231eb341292c62e8aa092 Mon Sep 17 00:00:00 2001 From: Stephen Sawchuk Date: Mon, 6 Apr 2015 10:44:03 -0400 Subject: [PATCH] integrate google-auth-library --- .jshintrc | 10 +- docs/site/components/docs/docs.html | 12 +- lib/bigquery/index.js | 24 +- lib/common/util.js | 314 +++++++--- lib/datastore/dataset.js | 13 +- lib/datastore/index.js | 16 +- lib/index.js | 44 +- lib/pubsub/index.js | 37 +- lib/storage/index.js | 45 +- package.json | 4 +- regression/index.js | 50 ++ test/common/util.js | 927 +++++++++++++++++----------- test/pubsub/index.js | 16 +- 13 files changed, 957 insertions(+), 555 deletions(-) create mode 100644 regression/index.js diff --git a/.jshintrc b/.jshintrc index 42d4fa95a8f..4c934ed6055 100644 --- a/.jshintrc +++ b/.jshintrc @@ -15,5 +15,13 @@ "strict": true, "trailing": true, "undef": true, - "unused": true + "unused": true, + "globals": { + "describe": true, + "it": true, + "before": true, + "after": true, + "beforeEach": true, + "afterEach": true + } } diff --git a/docs/site/components/docs/docs.html b/docs/site/components/docs/docs.html index d72ecb41af3..86264557d39 100644 --- a/docs/site/components/docs/docs.html +++ b/docs/site/components/docs/docs.html @@ -43,14 +43,11 @@

$ npm install --save gcloud
var gcloud = require('gcloud');
-

- There are a couple of ways to use the gcloud module. -

If you are running your app on Google App Engine or Google Compute Engine, you won't need to worry about supplying connection configuration options to gcloud— we figure that out for you.

- However, if you're running your app elsewhere, you will need to provide this information. + However, if you're running your app elsewhere, you will need to provide project details to authenticate API requests.

// App Engine and Compute Engine @@ -58,10 +55,11 @@

// Elsewhere var gcloud = require('gcloud')({ + projectId: 'project-id', keyFilename: '/path/to/keyfile.json' });

- In any environment, you are free to provide these and other default properties, which eventually will be passed to the gcloud sub-modules (Datastore, Storage, etc.). + The full set of options which can be passed to gcloud and sub-modules are outlined here.

@@ -85,7 +83,7 @@

Datastore Overview

var dataset = gcloud.datastore.dataset({ - projectId: 'myProject', + projectId: 'project-id', keyFilename: '/path/to/keyfile.json' });

@@ -103,7 +101,7 @@

Pub/Sub Overview

var pubsub = gcloud.pubsub({ - projectId: 'myProject', + projectId: 'project-id', keyFilename: '/path/to/keyfile.json' });

diff --git a/lib/bigquery/index.js b/lib/bigquery/index.js index 7c13b99de04..3ffa2c9cf9d 100644 --- a/lib/bigquery/index.js +++ b/lib/bigquery/index.js @@ -63,16 +63,21 @@ var BIGQUERY_BASE_URL = 'https://www.googleapis.com/bigquery/v2/projects/'; var SCOPES = ['https://www.googleapis.com/auth/bigquery']; /** - * The example below will demonstrate the different usage patterns your app may + * The examples below will demonstrate the different usage patterns your app may * need to support to retrieve a BigQuery object. * + * The full set of options that can be passed to BigQuery are + * [outlined here](#/docs/?method=gcloud). + * * @alias module:bigquery * @constructor * * @example * var gcloud = require('gcloud'); * + * //- * // Providing configuration details up-front. + * //- * var myProject = gcloud({ * keyFilename: '/path/to/keyfile.json', * projectId: 'my-project' @@ -80,19 +85,13 @@ var SCOPES = ['https://www.googleapis.com/auth/bigquery']; * * var bigquery = myProject.bigquery(); * - * + * //- * // Overriding default configuration details. - * var anotherBigQueryInstance = myProject.bigquery({ + * //- + * var bigquery = myProject.bigquery({ * keyFilename: '/path/to/another/keyfile.json' * }); * - * - * // Not using a default configuration. - * var myOtherProject = gcloud.bigquery({ - * keyFilename: '/path/to/keyfile.json', - * projectId: 'my-project' - * }); - * * //- * // In the following examples from this page and the other modules (Dataset, * // Table, etc.), we are going to be using a dataset from @@ -109,10 +108,11 @@ function BigQuery(options) { options = options || {}; - this.makeAuthorizedRequest_ = util.makeAuthorizedRequest({ + this.makeAuthorizedRequest_ = util.makeAuthorizedRequestFactory({ credentials: options.credentials, keyFile: options.keyFilename, - scopes: SCOPES + scopes: SCOPES, + email: options.email }); this.projectId = options.projectId; diff --git a/lib/common/util.js b/lib/common/util.js index 8f7a984cac9..d75de15747d 100644 --- a/lib/common/util.js +++ b/lib/common/util.js @@ -22,20 +22,19 @@ */ var extend = require('extend'); -var gsa = require('google-service-account'); +var GoogleAuth = require('google-auth-library'); var request = require('request'); -var util = require('util'); +var nodeutil = require('util'); var uuid = require('node-uuid'); -/** @const {number} Maximum amount of times to attempt refreshing a token. */ -var MAX_TOKEN_REFRESH_ATTEMPTS = 1; - /** @const {object} gcloud-node's package.json file. */ var PKG = require('../../package.json'); /** @const {string} User agent. */ var USER_AGENT = 'gcloud-node/' + PKG.version; +var util = module.exports; + /** * Extend a global configuration object with user options provided at the time * of sub-module instantiation. @@ -76,7 +75,7 @@ function extendGlobalConfig(globalConfig, overrides) { return extend(true, {}, options, overrides); } -module.exports.extendGlobalConfig = extendGlobalConfig; +util.extendGlobalConfig = extendGlobalConfig; /** * Wrap an array around a non-Array object. If given an Array, it is returned. @@ -103,7 +102,7 @@ function arrayize(input) { return input; } -module.exports.arrayize = arrayize; +util.arrayize = arrayize; /** * Format a string with values from the provided object. @@ -125,7 +124,7 @@ function format(template, args) { }); } -module.exports.format = format; +util.format = format; /** * No op. @@ -137,7 +136,7 @@ module.exports.format = format; */ function noop() {} -module.exports.noop = noop; +util.noop = noop; /** * Extend the native Error object. @@ -155,7 +154,7 @@ function ApiError(errorBody) { this.response = errorBody.response; } -util.inherits(ApiError, Error); +nodeutil.inherits(ApiError, Error); /** * Uniformly process an API response. @@ -194,7 +193,7 @@ function handleResp(err, resp, body, callback) { callback(null, body, resp); } -module.exports.handleResp = handleResp; +util.handleResp = handleResp; /** * Get the type of a value. @@ -252,7 +251,7 @@ function prop(name) { }; } -module.exports.prop = prop; +util.prop = prop; /** * Assign a value to a property in an Array iterator. @@ -268,7 +267,7 @@ function propAssign(prop, value) { }; } -module.exports.propAssign = propAssign; +util.propAssign = propAssign; /** * Check if an object is of the given type. @@ -285,7 +284,7 @@ function is(value, type) { return getType(value).toLowerCase() === type.toLowerCase(); } -module.exports.is = is; +util.is = is; /** * Convert an object into an array. @@ -305,7 +304,7 @@ function toArray(object) { return [].slice.call(object); } -module.exports.toArray = toArray; +util.toArray = toArray; /** * Take a Duplexify stream, fetch an authorized connection header, and create an @@ -407,13 +406,13 @@ function makeWritableStream(dup, options, onComplete) { }); } -module.exports.makeWritableStream = makeWritableStream; +util.makeWritableStream = makeWritableStream; /** * Returns an exponential distributed time to wait given the number of retries * that have been previously been attempted on the request. * - * @param {number} retryNumber - The number of retries previously attempted. + * @param {number} retryNumber - The number of retries previously attempted. * @return {number} An exponentially distributed time to wait E.g. for use with * exponential backoff. */ @@ -421,17 +420,17 @@ function getNextRetryWait(retryNumber) { return (Math.pow(2, retryNumber) * 1000) + Math.floor(Math.random() * 1000); } -module.exports.getNextRetryWait = getNextRetryWait; +util.getNextRetryWait = getNextRetryWait; /** * Returns true if the API request should be retried, given the error that was * given the first time the request was attempted. This is used for rate limit * related errors as well as intermittent server errors. * - * @param {error} err - The API error to check if it is appropriate to retry. + * @param {error} err - The API error to check if it is appropriate to retry. * @return {boolean} True if the API request should be retried, false otherwise. */ -function shouldRetry(err) { +function shouldRetryRequest(err) { if (err) { if ([429, 500, 503].indexOf(err.code) !== -1) { return true; @@ -452,102 +451,233 @@ function shouldRetry(err) { return false; } -module.exports.shouldRetryErr = shouldRetry; +util.shouldRetryRequest = shouldRetryRequest; -function makeAuthorizedRequest(config) { - var GAE_OR_GCE = !config || (!config.credentials && !config.keyFile); - var MAX_RETRIES = config && config.maxRetries || 3; - var autoRetry = !config || config.autoRetry !== false ? true : false; - var attemptedRetries = 0; - - var missingCredentialsError = new Error(); - missingCredentialsError.message = [ - 'A connection to gcloud must be established via either a `keyFilename` ', - 'property or a `credentials` object.', - '\n\n', - 'See the "Getting Started with gcloud" section for more information:', - '\n\n', - '\thttps://googlecloudplatform.github.io/gcloud-node/#/docs/', - '\n' - ].join(''); - - var authorize; - - if (config.customEndpoint) { - // Using a custom API override. Do not use `google-service-account` for - // authentication. (ex: connecting to a local Datastore server) - authorize = function(reqOpts, callback) { - callback(null, reqOpts); - }; +/** + * Create an Auth Client from Google Auth Library, used to get an access token + * for authenticating API requests. + * + * @param {object} config - Configuration object. + * @param {object=} config.credentials - Credentials object. + * @param {string=} config.email - Account email address, required for PEM/P12 + * usage. + * @param {string=} config.keyFile - Path to a .json, .pem, or .p12 keyfile. + * @param {array} config.scopes - Array of scopes required for the API. + * @param {function} callback - The callback function. + */ +function getAuthClient(config, callback) { + var googleAuth = new GoogleAuth(); + + if (config.keyFile) { + var authClient = new googleAuth.JWT(); + authClient.keyFile = config.keyFile; + authClient.email = config.email; + authClient.scopes = config.scopes; + addScope(null, authClient); + } else if (config.credentials) { + googleAuth.fromJSON(config.credentials, addScope); } else { - authorize = gsa(config); + googleAuth.getApplicationDefault(addScope); + } + + function addScope(err, authClient) { + if (err) { + callback(err); + return; + } + + if (authClient.createScopedRequired && authClient.createScopedRequired()) { + authClient = authClient.createScoped(config.scopes); + } + + callback(null, authClient); } +} - function makeRequest(reqOpts, callback) { - var tokenRefreshAttempts = 0; - reqOpts.headers = reqOpts.headers || {}; +util.getAuthClient = getAuthClient; - if (reqOpts.headers['User-Agent']) { - reqOpts.headers['User-Agent'] += '; ' + USER_AGENT; - } else { - reqOpts.headers['User-Agent'] = USER_AGENT; +/** + * Authenticate a request by extending its headers object with an access token. + * + * @param {object} config - Configuration object. + * @param {object=} config.credentials - Credentials object. + * @param {string=} config.email - Account email address, required for PEM/P12 + * usage. + * @param {string=} config.keyFile - Path to a .json, .pem, or .p12 keyfile. + * @param {array} config.scopes - Array of scopes required for the API. + * @param {object} reqOpts - HTTP request options. Its `headers` object is + * created or extended with a valid access token. + * @param {function} callback - The callback function. + */ +function authorizeRequest(config, reqOpts, callback) { + util.getAuthClient(config, function(err, authClient) { + if (err) { + // google-auth-library returns a "Could not load..." error if it can't get + // an access token. However, it's possible an API request doesn't need to + // be authenticated, e.g. when downloading a file from a public bucket. We + // consider this error a warning, and allow the request to go through + // without authorization, relying on the upstream API to return an error + // the user would find more helpful, should one occur. + if (err.message.indexOf('Could not load') === 0) { + callback(null, reqOpts); + } else { + callback(err); + } + return; } - function onAuthorizedRequest(err, authorizedReqOpts) { + authClient.getAccessToken(function(err, token) { if (err) { - if (GAE_OR_GCE && err.code === 'ENOTFOUND') { - // The metadata server wasn't found. The user must not actually be in - // a GAE or GCE environment. - throw missingCredentialsError; - } + callback(err); + return; + } - if (err.code === 401 && - ++tokenRefreshAttempts <= MAX_TOKEN_REFRESH_ATTEMPTS) { - authorize(reqOpts, onAuthorizedRequest); - return; + var authorizedReqOpts = extend(true, {}, reqOpts, { + headers: { + Authorization: 'Bearer ' + token } + }); - // For detecting Sign errors on io.js (1.x) (or node 0.11.x) - // E.g. errors in form: error:code:PEM routines:PEM_read_bio:error_name - var pemError = err.message && - err.message.indexOf('error:') !== -1; + callback(null, authorizedReqOpts); + }); + }); +} - if (err.message === 'SignFinal error' || pemError) { - err.message = [ - 'Your private key is in an unexpected format and cannot be used.', - 'Please try again with another private key.' - ].join(' '); - } +util.authorizeRequest = authorizeRequest; - (callback.onAuthorized || callback)(err); - return; +/** + * Get a function for making authorized requests. + * + * @param {object} config - Configuration object. + * @param {boolean=} config.autoRetry - Automatically retry requests if the + * response is related to rate limits or certain intermittent server errors. + * We will exponentially backoff subsequent requests by default. (default: + * true) + * @param {object=} config.credentials - Credentials object. + * @param {boolean=} config.customEndpoint - If true, just return the provided + * request options. Default: false. + * @param {string=} config.email - Account email address, required for PEM/P12 + * usage. + * @param {number=} config.maxRetries - Maximum number of automatic retries + * attempted before returning the error. (default: 3) + * @param {string=} config.keyFile - Path to a .json, .pem, or .p12 keyfile. + * @param {array} config.scopes - Array of scopes required for the API. + */ +function makeAuthorizedRequestFactory(config) { + config = config || {}; + + /** + * The returned function that will make an authorized request. + * + * @param {type} reqOpts - Request options in the format `request` expects. + * @param {object|function} options - Configuration object or callback + * function. + * @param {function=} options.onAuthorized - If provided, a request will not + * be made. Instead, this function is passed the error & authorized + * request options. + */ + function makeAuthorizedRequest(reqOpts, callback) { + if (config.customEndpoint) { + // Using a custom API override. Do not use `google-auth-library` for + // authentication. (ex: connecting to a local Datastore server) + if (callback.onAuthorized) { + callback.onAuthorized(null, reqOpts); + } else { + util.makeRequest(reqOpts, config, callback); } - function handleRateLimitResp(err, res, body) { - handleResp(err, res, body, function(err, body, resp) { - if (shouldRetry(err) && autoRetry && MAX_RETRIES > attemptedRetries) { - setTimeout(function() { - request(authorizedReqOpts, handleRateLimitResp); - }, getNextRetryWait(attemptedRetries++)); - } else { - callback(err, body, resp); - } - }); + return; + } + + util.authorizeRequest(config, reqOpts, function(err, authorizedReqOpts) { + if (err) { + (callback.onAuthorized || callback)(err); + return; } if (callback.onAuthorized) { callback.onAuthorized(null, authorizedReqOpts); } else { - request(authorizedReqOpts, handleRateLimitResp); + util.makeRequest(authorizedReqOpts, config, callback); } - } + }); + } + + makeAuthorizedRequest.getCredentials = function(callback) { + util.getAuthClient(config, function(err, authClient) { + if (err) { + callback(err); + return; + } + + authClient.authorize(function(err) { + if (err) { + callback(err); + return; + } + + callback(null, { + client_email: authClient.email, + private_key: authClient.key + }); + }); + }); + }; + + return makeAuthorizedRequest; +} + +util.makeAuthorizedRequestFactory = makeAuthorizedRequestFactory; + +/** + * Make a request through the `request` module with built-in error handling and + * exponential back off. + * + * @param {object} reqOpts - Request options in the format `request` expects. + * @param {object=} config - Configuration object. + * @param {boolean=} config.autoRetry - Automatically retry requests if the + * response is related to rate limits or certain intermittent server errors. + * We will exponentially backoff subsequent requests by default. (default: + * true) + * @param {number=} config.maxRetries - Maximum number of automatic retries + * attempted before returning the error. (default: 3) + * @param {function} callback - The callback function. + */ +function makeRequest(reqOpts, config, callback) { + if (util.is(config, 'function')) { + callback = config; + config = {}; + } + + config = config || {}; + + var MAX_RETRIES = config.maxRetries || 3; + var autoRetry = config.autoRetry !== false ? true : false; + var attemptedRetries = 0; - authorize(reqOpts, onAuthorizedRequest); + reqOpts.headers = reqOpts.headers || {}; + reqOpts.headers['User-Agent'] = USER_AGENT; + + function shouldRetry(err) { + return autoRetry && + MAX_RETRIES > attemptedRetries && + util.shouldRetryRequest(err); } - makeRequest.getCredentials = authorize.getCredentials; + function makeRateLimitedRequest() { + request(reqOpts, function(err, resp, body) { + util.handleResp(err, resp, body, function(err, body, resp) { + if (shouldRetry(err)) { + var delay = util.getNextRetryWait(attemptedRetries++); + setTimeout(makeRateLimitedRequest, delay); + } else { + callback(err || null, body, resp); + } + }); + }); + } - return makeRequest; + makeRateLimitedRequest(); } -module.exports.makeAuthorizedRequest = makeAuthorizedRequest; +util.makeRequest = makeRequest; diff --git a/lib/datastore/dataset.js b/lib/datastore/dataset.js index a268914f1c4..4ed176c659f 100644 --- a/lib/datastore/dataset.js +++ b/lib/datastore/dataset.js @@ -66,6 +66,9 @@ var SCOPES = [ * Interact with a dataset from the * [Google Cloud Datastore](https://developers.google.com/datastore/). * + * The full set of options that can be passed to this method are + * [outlined here](#/docs/?method=gcloud). + * * @constructor * @alias module:datastore/dataset * @mixes module:datastore/request @@ -73,11 +76,6 @@ var SCOPES = [ * @param {object=} options - Configuration object. * @param {string=} options.projectId - Dataset ID. This is your project ID from * the Google Developers Console. - * @param {string=} options.keyFilename - Full path to the JSON key downloaded - * from the Google Developers Console. Alternatively, you may provide a - * `credentials` object. - * @param {object=} options.credentials - Credentials object, used in place of - * a `keyFilename`. * @param {string=} options.apiEndpoint - Override the default API endpoint used * to reach Datastore. This is useful for connecting to your local Datastore * server (usually "http://localhost:8080"). @@ -104,11 +102,12 @@ function Dataset(options) { options = options || {}; - this.makeAuthorizedRequest_ = util.makeAuthorizedRequest({ + this.makeAuthorizedRequest_ = util.makeAuthorizedRequestFactory({ customEndpoint: typeof options.apiEndpoint !== 'undefined', credentials: options.credentials, keyFile: options.keyFilename, - scopes: SCOPES + scopes: SCOPES, + email: options.email }); if (options.apiEndpoint && options.apiEndpoint.indexOf('http') !== 0) { diff --git a/lib/datastore/index.js b/lib/datastore/index.js index c3385e1cd4c..26c7d6a8c59 100644 --- a/lib/datastore/index.js +++ b/lib/datastore/index.js @@ -65,26 +65,22 @@ var Dataset = require('./dataset'); * @example * var gcloud = require('gcloud'); * + * //- * // Providing configuration details up-front. + * //- * var myProject = gcloud({ * keyFilename: '/path/to/keyfile.json', * projectId: 'my-project' * }); * - * var dataset = myProject.datastore.dataset(); - * + * var dataset = gcloud.dataset(); * + * //- * // Overriding default configuration details. - * var anotherDataset = myProject.datastore.dataset({ + * //- + * var dataset = myProject.datastore.dataset({ * keyFilename: '/path/to/another/keyfile.json' * }); - * - * - * // Not using a default configuration. - * var myOtherProject = gcloud.datastore.dataset({ - * keyFilename: '/path/to/keyfile.json', - * projectId: 'my-project' - * }); */ function Datastore(config) { this.config = config || {}; diff --git a/lib/index.js b/lib/index.js index 582af258c6f..fd1df902601 100644 --- a/lib/index.js +++ b/lib/index.js @@ -72,22 +72,50 @@ var util = require('./common/util.js'); * your provided configuration will remain isolated to the returned gcloud * module. * + * To authenticate API requests, + * [google-auth-library](https://github.com/google/google-auth-library-nodejs) + * is used to detect the environment your project is running in. Use the + * following guide to determine the appropriate configuration. + * + *

Google Compute Engine

+ * + * - No configuration necessary! + * + *

Google App Engine Production

+ * + * - No configuration necessary! + * + *

Other

+ + * - Provide `config.projectId` + * - Use one of the following: + * - `config.credentials` object containing `client_email` and `private_key` + * properties. + * - `config.keyFilename` path to a .json, .pem, or .p12 key file. + * - `GOOGLE_APPLICATION_CREDENTIALS` environment variable with a full path + * to your key file. + * + * **Note**: When using a .pem or .p12 key file, `config.email` is also + * required. + * * @alias module:gcloud * @constructor * * @param {object} config - Connection configuration options. - * @param {string=} config.keyFilename - Full path to the JSON key downloaded - * from the Google Developers Console. Alternatively, you may provide a - * `credentials` object. + * @param {string=} config.keyFilename - Full path to the a .json, .pem, or .p12 + * key downloaded from the Google Developers Console. NOTE: .pem and .p12 + * require you to specify `config.email` as well. + * @param {string=} config.email - Account email address. Required when using a + * .pem or .p12 keyFilename. * @param {object=} config.credentials - Credentials object. * @param {string} config.credentials.client_email * @param {string} config.credentials.private_key - * @param {boolean} config.autoRetry - Automatically retry requests if the + * @param {boolean=} config.autoRetry - Automatically retry requests if the * response is related to rate limits or certain intermittent server errors. - * (default: true). Recommended is true. We will exponentially backoff - * subsequent requests by default. - * @param {number} config.maxRetries - Max number of auto retries to attempt - * before returning the error. (default: 3). + * We will exponentially backoff subsequent requests by default. (default: + * true) + * @param {number=} config.maxRetries - Maximum number of automatic retries + * attempted before returning the error. (default: 3) * * @example * var gcloud = require('gcloud')({ diff --git a/lib/pubsub/index.js b/lib/pubsub/index.js index 14fa0ae7b8c..06a4473aec9 100644 --- a/lib/pubsub/index.js +++ b/lib/pubsub/index.js @@ -64,6 +64,9 @@ var SCOPES = [ * subject to any SLA or deprecation policy. Request to be whitelisted to use * it by filling the [Limited Preview application form](http://goo.gl/sO0wTu). * + * The full set of options that can be passed to this method are + * [outlined here](#/docs/?method=gcloud). + * * @constructor * @alias module:pubsub * @@ -78,41 +81,31 @@ var SCOPES = [ * @example * var gcloud = require('gcloud'); * - * // From Google Compute Engine and Google App Engine: - * - * // Access `pubsub` through the `gcloud` module directly. - * var pubsub = gcloud.pubsub(); - * - * // Elsewhere: - * - * // Provide configuration details up-front. + * //- + * // Providing configuration details up-front. + * //- * var myProject = gcloud({ * keyFilename: '/path/to/keyfile.json', * projectId: 'my-project' * }); * - * var pubsub = myProject.pubsub(); - * - * - * // Override default configuration details. - * var anotherPubsubConnection = myProject.pubsub({ - * keyFilename: '/path/to/another/keyfile.json', - * }); - * + * var pubsub = gcloud.pubsub(); * - * // Specify all options at instantiation. - * var pubsub = gcloud.pubsub({ - * keyFilename: '/path/to/keyfile.json', - * projectId: 'my-project' + * //- + * // Overriding default configuration details. + * //- + * var pubsub = myProject.pubsub({ + * keyFilename: '/path/to/another/keyfile.json' * }); */ function PubSub(options) { options = options || {}; - this.makeAuthorizedRequest_ = util.makeAuthorizedRequest({ + this.makeAuthorizedRequest_ = util.makeAuthorizedRequestFactory({ credentials: options.credentials, keyFile: options.keyFilename, - scopes: SCOPES + scopes: SCOPES, + email: options.email }); this.projectId = options.projectId; diff --git a/lib/storage/index.js b/lib/storage/index.js index 2fb51e82a12..39faa82c3e2 100644 --- a/lib/storage/index.js +++ b/lib/storage/index.js @@ -54,15 +54,18 @@ var STORAGE_BASE_URL = 'https://www.googleapis.com/storage/v1/b'; * there is an equivalent static and instance method. While they are classes, * they can be instantiated without use of the `new` keyword. * - * @param {object} config - Configuration object. + * @param {object} options - Configuration object. */ /** * To access your Cloud Storage buckets, you will use the `bucket` function * returned from this `storage` object. * - * The example below will demonstrate the different usage patterns your app may + * The examples below will demonstrate the different usage patterns your app may * need to connect to `gcloud` and access your bucket. * + * The full set of options that can be passed to this method are + * [outlined here](#/docs/?method=gcloud). + * * @alias module:storage * @constructor * @@ -70,47 +73,37 @@ var STORAGE_BASE_URL = 'https://www.googleapis.com/storage/v1/b'; * var gcloud = require('gcloud'); * * //- - * // From Google Compute Engine and Google App Engine. + * // Providing configuration details up-front. * //- - * - * // Access `storage` through the `gcloud` module directly. - * var storage = gcloud.storage(); - * var musicBucket = storage.bucket('music'); - * - * //- - * // Elsewhere. - * //- - * - * // Provide configuration details up-front. * var myProject = gcloud({ * keyFilename: '/path/to/keyfile.json', * projectId: 'my-project' * }); * - * // Use default configuration details. * var storage = myProject.storage(); - * var albums = storage.bucket('albums'); - * var photos = storage.bucket('photos'); - * * - * // Override default configuration details. + * //- + * // Overriding default configuration details. + * //- * var storage = myProject.storage({ * keyFilename: '/path/to/another/keyfile.json' * }); - * var records = storage.bucket('records'); */ -function Storage(config) { +function Storage(options) { if (!(this instanceof Storage)) { - return new Storage(config); + return new Storage(options); } - this.makeAuthorizedRequest_ = util.makeAuthorizedRequest({ - credentials: config.credentials, - keyFile: config.keyFilename, - scopes: SCOPES + options = options || {}; + + this.makeAuthorizedRequest_ = util.makeAuthorizedRequestFactory({ + credentials: options.credentials, + keyFile: options.keyFilename, + scopes: SCOPES, + email: options.email }); - this.projectId = config.projectId; + this.projectId = options.projectId; } /** diff --git a/package.json b/package.json index 0722fa637da..a50b0a9a72b 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,7 @@ "duplexify": "^3.2.0", "extend": "^2.0.0", "fast-crc32c": "^0.1.3", - "google-service-account": "^1.0.3", + "google-auth-library": "^0.9.4", "mime-types": "^2.0.8", "node-uuid": "^1.4.2", "once": "^1.3.1", @@ -66,7 +66,7 @@ "devDependencies": { "bytebuffer": "^3.5.4", "coveralls": "^2.11.2", - "dox": "^0.6.1", + "dox": "^0.7.0", "istanbul": "^0.3.5", "jshint": "^2.6.0", "mocha": "^2.1.0", diff --git a/regression/index.js b/regression/index.js new file mode 100644 index 00000000000..85ee0b232e7 --- /dev/null +++ b/regression/index.js @@ -0,0 +1,50 @@ +/** + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +var assert = require('assert'); +var env = require('./env'); +var gcloud = require('../lib'); + +// Test used to confirm we can perform a successful API operation. +function canConnect(config, callback) { + gcloud.storage(config).getBuckets(callback); +} + +describe('environment', function() { + it('should connect with credentials object', canConnect.bind(null, { + projectId: env.projectId, + credentials: require(env.keyFilename) + })); + + it('should connect from a JSON keyFilename', canConnect.bind(null, { + projectId: env.projectId, + keyFilename: env.keyFilename + })); + + it('should connect from environment variable', function(done) { + var ENV_VAR = 'GOOGLE_APPLICATION_CREDENTIALS'; + + process.env[ENV_VAR] = env.keyFilename; + + canConnect({ projectId: env.projectId }, function(err) { + assert.ifError(err); + delete process.env[ENV_VAR]; + done(); + }); + }); +}); diff --git a/test/common/util.js b/test/common/util.js index 7d939559a33..9d9c6041a82 100644 --- a/test/common/util.js +++ b/test/common/util.js @@ -20,35 +20,45 @@ var assert = require('assert'); var duplexify = require('duplexify'); -var gsa = require('google-service-account'); +var extend = require('extend'); +var googleAuthLibrary = require('google-auth-library'); var mockery = require('mockery'); var request = require('request'); var stream = require('stream'); -var gsa_Override; -function fakeGsa() { - var args = [].slice.apply(arguments); - var results = (gsa_Override || gsa).apply(null, args); - return results || { getCredentials: function() {} }; +var googleAuthLibrary_Override; +function fakeGoogleAuthLibrary() { + return (googleAuthLibrary_Override || googleAuthLibrary) + .apply(null, arguments); } var request_Override; function fakeRequest() { - var args = [].slice.apply(arguments); - return (request_Override || request).apply(null, args); + return (request_Override || request).apply(null, arguments); } describe('common/util', function() { var util; + var utilOverrides = {}; before(function() { - mockery.registerMock('google-service-account', fakeGsa); + mockery.registerMock('google-auth-library', fakeGoogleAuthLibrary); mockery.registerMock('request', fakeRequest); mockery.enable({ useCleanCache: true, warnOnUnregistered: false }); util = require('../../lib/common/util'); + var util_Cached = extend(true, {}, util); + + // Override all util methods, allowing them to be mocked. Overrides are + // removed before each test. + Object.keys(util).forEach(function(utilMethod) { + util[utilMethod] = function() { + return (utilOverrides[utilMethod] || util_Cached[utilMethod]) + .apply(this, arguments); + }; + }); }); after(function() { @@ -57,8 +67,9 @@ describe('common/util', function() { }); beforeEach(function() { - gsa_Override = null; + googleAuthLibrary_Override = null; request_Override = null; + utilOverrides = {}; }); describe('arrayize', function() { @@ -287,472 +298,441 @@ describe('common/util', function() { }); }); - describe('makeAuthorizedRequest', function() { - it('should pass configuration to gsa', function(done) { - var config = { keyFile: 'key', scopes: [1, 2] }; + describe('getAuthClient', function() { + it('should use google-auth-library', function() { + var googleAuthLibraryCalled = false; - gsa_Override = function(cfg) { - assert.deepEqual(cfg, config); - done(); + googleAuthLibrary_Override = function() { + googleAuthLibraryCalled = true; + return { + getApplicationDefault: util.noop + }; }; - util.makeAuthorizedRequest(config); + util.getAuthClient({}); + + assert.strictEqual(googleAuthLibraryCalled, true); }); - it('should not authenticate requests with a custom API', function(done) { - var makeRequest = util.makeAuthorizedRequest({ customEndpoint: true }); + it('should create a JWT auth client from a keyFile', function(done) { + var jwt = {}; - var gsaCalled = false; - gsa_Override = function() { - gsaCalled = true; + googleAuthLibrary_Override = function() { + return { + JWT: function() { return jwt; } + }; }; - makeRequest({}, { - onAuthorized: function(err) { - assert.ifError(err); - assert.strictEqual(gsaCalled, false); - done(); - } + var config = { + keyFile: 'key.json', + email: 'example@example.com', + scopes: ['dev.scope'] + }; + + util.getAuthClient(config, function(err, authClient) { + assert.ifError(err); + + assert.equal(jwt.keyFile, config.keyFile); + assert.equal(jwt.email, config.email); + assert.deepEqual(jwt.scopes, config.scopes); + + assert.deepEqual(authClient, jwt); + + done(); }); }); - it('should return gsa.getCredentials function', function() { - var getCredentials = util.makeAuthorizedRequest({}).getCredentials; - assert.equal(typeof getCredentials, 'function'); - }); + it('should create an auth client from credentials', function(done) { + var credentialsSet; - describe('makeRequest', function() { - it('should add a user agent onto headers', function(done) { - gsa_Override = function() { - return function authorize(reqOpts) { - assert(reqOpts.headers['User-Agent'].indexOf('gcloud') > -1); - done(); - }; + googleAuthLibrary_Override = function() { + return { + fromJSON: function(credentials, callback) { + credentialsSet = credentials; + callback(null, {}); + } }; + }; - var makeRequest = util.makeAuthorizedRequest({}); - makeRequest({}); + var config = { + credentials: { a: 'b', c: 'd' } + }; + + util.getAuthClient(config, function() { + assert.deepEqual(credentialsSet, config.credentials); + done(); }); + }); - it('should extend an existing user agent', function(done) { - gsa_Override = function() { - return function authorize(reqOpts) { - var index = reqOpts.headers['User-Agent'].indexOf('test; gcloud'); - assert.equal(index, 0); - done(); - }; + it('should create an auth client from magic', function(done) { + googleAuthLibrary_Override = function() { + return { + getApplicationDefault: function(callback) { + callback(null, {}); + } }; + }; - var makeRequest = util.makeAuthorizedRequest({}); - makeRequest({ headers: { 'User-Agent': 'test' } }); - }); + util.getAuthClient({}, done); + }); - it('should execute callback with error', function(done) { - var error = new Error('Error.'); + it('should scope an auth client if necessary', function(done) { + var config = { + scopes: ['a.scope', 'b.scope'] + }; - gsa_Override = function() { - return function authorize(reqOpts, callback) { - callback(error); - }; + var fakeAuthClient = { + createScopedRequired: function() { + return true; + }, + createScoped: function(scopes) { + assert.deepEqual(scopes, config.scopes); + return fakeAuthClient; + }, + getAccessToken: function() {} + }; + + googleAuthLibrary_Override = function() { + return { + getApplicationDefault: function(callback) { + callback(null, fakeAuthClient); + } }; + }; - var makeRequest = util.makeAuthorizedRequest({}); - makeRequest({}, function(err) { - assert.equal(err, error); - done(); - }); - }); + util.getAuthClient(config, done); + }); + }); - it('should throw if not GCE/GAE & missing credentials', function() { - gsa_Override = function() { - return function authorize(reqOpts, callback) { - // Simulate the metadata server not existing. - callback({ code: 'ENOTFOUND' }); - }; - }; + describe('authorizeRequest', function() { + it('should get an auth client', function(done) { + var config = { a: 'b', c: 'd' }; - assert.throws(function() { - // Don't provide a keyFile or credentials object. - var connectionConfig = {}; - var makeRequest = util.makeAuthorizedRequest(connectionConfig); - makeRequest({}, util.noop); - }, /A connection to gcloud must be established/); - }); + utilOverrides.getAuthClient = function(cfg) { + assert.deepEqual(cfg, config); + done(); + }; - it('should handle malformed key response', function(done) { - var makeRequest = util.makeAuthorizedRequest({ - credentials: { - client_email: 'invalid@email', - private_key: 'invalid-key' - } - }); + util.authorizeRequest(config); + }); - makeRequest({}, function (err) { - var errorMessage = [ - 'Your private key is in an unexpected format and cannot be used.', - 'Please try again with another private key.' - ].join(' '); - assert.equal(err.message, errorMessage); - done(); - }); + it('should ignore "Could not load" error from google-auth', function(done) { + var reqOpts = { a: 'b', c: 'd' }; + var couldNotLoadError = new Error('Could not load'); + + utilOverrides.getAuthClient = function(config, callback) { + callback(couldNotLoadError); + }; + + util.authorizeRequest({}, reqOpts, function(err, authorizedReqOpts) { + assert.ifError(err); + assert.deepEqual(reqOpts, authorizedReqOpts); + done(); }); + }); - it('should try to reconnect if token invalid', function(done) { - var attempts = 0; - var expectedAttempts = 2; - var error = { code: 401 }; + it('should return an error to the callback', function(done) { + var error = new Error('Error.'); - gsa_Override = function() { - return function authorize(reqOpts, callback) { - attempts++; - callback(error); - }; - }; + utilOverrides.getAuthClient = function(config, callback) { + callback(error); + }; - var makeRequest = util.makeAuthorizedRequest({}); - makeRequest({}, function (err) { - assert.equal(attempts, expectedAttempts); - assert.equal(err, error); - done(); - }); + util.authorizeRequest({}, {}, function(err) { + assert.deepEqual(err, error); + done(); }); + }); - it('should execute the onauthorized callback', function(done) { - gsa_Override = function() { - return function authorize(reqOpts, callback) { - callback(); - }; - }; + it('should get an access token', function(done) { + var fakeAuthClient = { + getAccessToken: function() { + done(); + } + }; + + utilOverrides.getAuthClient = function(config, callback) { + callback(null, fakeAuthClient); + }; + + util.authorizeRequest(); + }); - var makeRequest = util.makeAuthorizedRequest({}); - makeRequest({}, { onAuthorized: done }); + it('should return an access token error to callback', function(done) { + var error = new Error('Error.'); + + var fakeAuthClient = { + getAccessToken: function(callback) { + callback(error); + } + }; + + utilOverrides.getAuthClient = function(config, callback) { + callback(null, fakeAuthClient); + }; + + util.authorizeRequest({}, {}, function(err) { + assert.deepEqual(err, error); + done(); }); + }); - it('should execute the onauthorized callback with error', function(done) { - var error = new Error('Error.'); + it('should extend the request options with token', function(done) { + var token = 'abctoken'; - gsa_Override = function() { - return function authorize(reqOpts, callback) { - callback(error); - }; - }; + var reqOpts = { + uri: 'a', + headers: { + a: 'b', + c: 'd' + } + }; - var makeRequest = util.makeAuthorizedRequest({}); - makeRequest({}, { - onAuthorized: function(err) { - assert.equal(err, error); - done(); - } - }); + var expectedAuthorizedReqOpts = extend(true, {}, reqOpts, { + headers: { + Authorization: 'Bearer ' + token + } }); - it('should make the authorized request', function(done) { - var authorizedReqOpts = { a: 'b', c: 'd' }; + var fakeAuthClient = { + getAccessToken: function(callback) { + callback(null, token); + } + }; - gsa_Override = function() { - return function authorize(reqOpts, callback) { - callback(null, authorizedReqOpts); - }; - }; + utilOverrides.getAuthClient = function(config, callback) { + callback(null, fakeAuthClient); + }; - request_Override = function(reqOpts) { - assert.deepEqual(reqOpts, authorizedReqOpts); - done(); - }; + util.authorizeRequest({}, reqOpts, function(err, authorizedReqOpts) { + assert.ifError(err); + + assert.deepEqual(authorizedReqOpts, expectedAuthorizedReqOpts); - var makeRequest = util.makeAuthorizedRequest({}); - makeRequest({}, assert.ifError); + done(); }); + }); + }); - it('should retry rate limit requests by default', function(done) { - var attemptedRetries = 0; - var error = new Error('Rate Limit Error.'); - error.code = 429; // Rate limit error + describe('makeAuthorizedRequestFactory', function() { + it('should return a function', function() { + assert.equal(typeof util.makeAuthorizedRequestFactory(), 'function'); + }); - var authorizedReqOpts = { a: 'b', c: 'd' }; + describe('customEndpoint (no authorization attempted)', function() { + var makeAuthorizedRequest; - var old_setTimeout = setTimeout; - setTimeout = function(callback, time) { - var MIN_TIME = (Math.pow(2, attemptedRetries) * 1000); - var MAX_TIME = (Math.pow(2, attemptedRetries) * 1000) + 1000; - assert(time >= MIN_TIME && time <= MAX_TIME); - attemptedRetries++; - callback(); // make the request again - }; + beforeEach(function() { + makeAuthorizedRequest = util.makeAuthorizedRequestFactory({ + customEndpoint: true + }); + }); - gsa_Override = function() { - return function authorize(reqOpts, callback) { - callback(null, authorizedReqOpts); - }; - }; + it('should pass options back to onAuthorized callback', function(done) { + var reqOpts = { a: 'b', c: 'd' }; - request_Override = function(reqOpts, callback) { - if (attemptedRetries === 3) { - setTimeout = old_setTimeout; + makeAuthorizedRequest(reqOpts, { + onAuthorized: function(err, authorizedReqOpts) { + assert.ifError(err); + assert.deepEqual(reqOpts, authorizedReqOpts); done(); - } else { - callback(error); // this callback should check for rate limits } + }); + }); + + it('should not authenticate requests with a custom API', function(done) { + var reqOpts = { a: 'b', c: 'd' }; + + utilOverrides.makeRequest = function(rOpts) { + assert.deepEqual(rOpts, reqOpts); + done(); }; - var makeRequest = util.makeAuthorizedRequest({}); - makeRequest({}, assert.ifError); + makeAuthorizedRequest(reqOpts, assert.ifError); }); + }); - it('should retry rate limits 3x on 429, 500, 503', function(done) { - var attemptedRetries = 0; - var codes = [429, 503, 500, 'done']; - var error = new Error('Rate Limit Error.'); - error.code = codes[0]; // Rate limit error - - var authorizedReqOpts = { a: 'b', c: 'd' }; + describe('needs authorization', function() { + it('should pass correct arguments to authorizeRequest', function(done) { + var config = { a: 'b', c: 'd' }; + var reqOpts = { e: 'f', g: 'h' }; - var old_setTimeout = setTimeout; - setTimeout = function(callback, time) { - var MIN_TIME = (Math.pow(2, attemptedRetries) * 1000); - var MAX_TIME = (Math.pow(2, attemptedRetries) * 1000) + 1000; - assert(time >= MIN_TIME && time <= MAX_TIME); - attemptedRetries++; - error.code = codes[attemptedRetries]; // test a new code - callback(); // make the request again + utilOverrides.authorizeRequest = function(cfg, rOpts) { + assert.deepEqual(cfg, config); + assert.deepEqual(rOpts, reqOpts); + done(); }; - gsa_Override = function() { - return function authorize(reqOpts, callback) { - callback(null, authorizedReqOpts); + var makeAuthorizedRequest = util.makeAuthorizedRequestFactory(config); + makeAuthorizedRequest(reqOpts); + }); + + describe('authorization errors', function() { + var error = new Error('Error.'); + + beforeEach(function() { + utilOverrides.authorizeRequest = function(cfg, rOpts, callback) { + callback(error); }; - }; + }); - request_Override = function(reqOpts, callback) { - callback(error); // this callback should check for rate limits - }; + it('should invoke the callback with error', function(done) { + var makeAuthorizedRequest = util.makeAuthorizedRequestFactory(); + makeAuthorizedRequest({}, function(err) { + assert.deepEqual(err, error); + done(); + }); + }); - var makeRequest = util.makeAuthorizedRequest({}); - makeRequest({}, function(err) { - setTimeout = old_setTimeout; - assert.equal(err, error); - assert.equal(err.code, 'done'); - done(); + it('should invoke the onAuthorized handler with error', function(done) { + var makeAuthorizedRequest = util.makeAuthorizedRequestFactory(); + makeAuthorizedRequest({}, { + onAuthorized: function(err) { + assert.deepEqual(err, error); + done(); + } + }); }); }); - it('should retry rate limits on API errors', function(done) { - var attemptedRetries = 0; - var codes = [429, 503, 500, 'done']; - var error = new Error('Rate Limit Error.'); - error.code = codes[0]; // Rate limit error + describe('authorization success', function() { + var reqOpts = { a: 'b', c: 'd' }; + + it('should return the authorized request to callback', function(done) { + utilOverrides.authorizeRequest = function(cfg, rOpts, callback) { + callback(null, rOpts); + }; - var authorizedReqOpts = { a: 'b', c: 'd' }; + var makeAuthorizedRequest = util.makeAuthorizedRequestFactory(); + makeAuthorizedRequest(reqOpts, { + onAuthorized: function(err, authorizedReqOpts) { + assert.deepEqual(authorizedReqOpts, reqOpts); + done(); + } + }); + }); - var old_setTimeout = setTimeout; - setTimeout = function(callback, time) { - var MIN_TIME = (Math.pow(2, attemptedRetries) * 1000); - var MAX_TIME = (Math.pow(2, attemptedRetries) * 1000) + 1000; - assert(time >= MIN_TIME && time <= MAX_TIME); - attemptedRetries++; - error.code = codes[attemptedRetries]; // test a new code - callback(); // make the request again - }; + it('should make request with correct options', function(done) { + var config = { a: 'b', c: 'd' }; - gsa_Override = function() { - return function authorize(reqOpts, callback) { - callback(null, authorizedReqOpts); + utilOverrides.authorizeRequest = function(cfg, rOpts, callback) { + callback(null, rOpts); }; - }; - request_Override = function(reqOpts, callback) { - callback(null, null, { error: error }); - }; + utilOverrides.makeRequest = function(authorizedReqOpts, cfg, cb) { + assert.deepEqual(authorizedReqOpts, reqOpts); + assert.deepEqual(cfg, config); + cb(); + }; - var makeRequest = util.makeAuthorizedRequest({}); - makeRequest({}, function(err) { - setTimeout = old_setTimeout; - assert.equal(err.message, 'Rate Limit Error.'); - assert.equal(err.code, 'done'); - done(); + var makeAuthorizedRequest = util.makeAuthorizedRequestFactory(config); + makeAuthorizedRequest(reqOpts, done); }); }); + }); - it('should retry rate limits on rateLimitExceeded', function(done) { - var attemptedRetries = 0; - var error = new Error('Rate Limit Error.'); - error.code = 403; // not a rate limit code! - error.errors = [{ reason: 'rateLimitExceeded' }]; + describe('getCredentials', function() { + var fakeAuthClient = { + email: 'fake-email@example.com', + key: 'fake-key', - var authorizedReqOpts = { a: 'b', c: 'd' }; + authorize: function(callback) { callback(); } + }; + var config = { a: 'b', c: 'd' }; - var old_setTimeout = setTimeout; - setTimeout = function(callback, time) { - var MIN_TIME = (Math.pow(2, attemptedRetries) * 1000); - var MAX_TIME = (Math.pow(2, attemptedRetries) * 1000) + 1000; - assert(time >= MIN_TIME && time <= MAX_TIME); - attemptedRetries++; - callback(); // make the request again + it('should return getCredentials method', function() { + utilOverrides.getAuthClient = function(config, callback) { + callback(null, fakeAuthClient); }; - gsa_Override = function() { - return function authorize(reqOpts, callback) { - callback(null, authorizedReqOpts); - }; - }; + var makeAuthorizedRequest = + util.makeAuthorizedRequestFactory(config, assert.ifError); - request_Override = function(reqOpts, callback) { - callback(null, null, { error: error }); - }; + assert.equal(typeof makeAuthorizedRequest.getCredentials, 'function'); + }); - var makeRequest = util.makeAuthorizedRequest({}); - makeRequest({}, function(err) { - setTimeout = old_setTimeout; - assert.equal(attemptedRetries, 3); - assert.equal(err.message, 'Rate Limit Error.'); + it('should pass config to getAuthClient', function(done) { + utilOverrides.getAuthClient = function(cfg) { + assert.deepEqual(cfg, config); done(); - }); - }); + }; - it('should retry rate limits on userRateLimitExceeded', function(done) { - var attemptedRetries = 0; - var error = new Error('Rate Limit Error.'); - error.code = 403; // not a rate limit code! - error.errors = [{ reason: 'userRateLimitExceeded' }]; + var makeAuthorizedRequest = + util.makeAuthorizedRequestFactory(config, assert.ifError); - var authorizedReqOpts = { a: 'b', c: 'd' }; + makeAuthorizedRequest.getCredentials(); + }); - var old_setTimeout = setTimeout; - setTimeout = function(callback, time) { - var MIN_TIME = (Math.pow(2, attemptedRetries) * 1000); - var MAX_TIME = (Math.pow(2, attemptedRetries) * 1000) + 1000; - assert(time >= MIN_TIME && time <= MAX_TIME); - attemptedRetries++; - callback(); // make the request again - }; + it('should execute callback with error', function(done) { + var error = new Error('Error.'); - gsa_Override = function() { - return function authorize(reqOpts, callback) { - callback(null, authorizedReqOpts); - }; + utilOverrides.getAuthClient = function(config, callback) { + callback(error); }; - request_Override = function(reqOpts, callback) { - callback(null, null, { error: error }); - }; + var makeAuthorizedRequest = + util.makeAuthorizedRequestFactory(config, assert.ifError); - var makeRequest = util.makeAuthorizedRequest({}); - makeRequest({}, function(err) { - setTimeout = old_setTimeout; - assert.equal(attemptedRetries, 3); - assert.equal(err.message, 'Rate Limit Error.'); + makeAuthorizedRequest.getCredentials(function(err) { + assert.deepEqual(err, error); done(); }); }); - it('should retry rate limits 3x by default', function(done) { - var attemptedRetries = 0; - var error = new Error('Rate Limit Error.'); - error.code = 429; // Rate limit error - - var authorizedReqOpts = { a: 'b', c: 'd' }; - - var old_setTimeout = setTimeout; - setTimeout = function(callback, time) { - var MIN_TIME = (Math.pow(2, attemptedRetries) * 1000); - var MAX_TIME = (Math.pow(2, attemptedRetries) * 1000) + 1000; - assert(time >= MIN_TIME && time <= MAX_TIME); - attemptedRetries++; - callback(); // make the request again + it('should authorize the connection', function(done) { + fakeAuthClient.authorize = function(callback) { + callback(); }; - gsa_Override = function() { - return function authorize(reqOpts, callback) { - callback(null, authorizedReqOpts); - }; + utilOverrides.getAuthClient = function(config, callback) { + callback(null, fakeAuthClient); }; - request_Override = function(reqOpts, callback) { - callback(error); // this callback should check for rate limits - }; + var makeAuthorizedRequest = + util.makeAuthorizedRequestFactory(config, assert.ifError); - var makeRequest = util.makeAuthorizedRequest({}); - makeRequest({}, function(err) { - setTimeout = old_setTimeout; - assert.equal(attemptedRetries, 3); - assert.equal(err, error); - done(); - }); + makeAuthorizedRequest.getCredentials(done); }); - it('should retry rate limits by maxRetries if provided', function(done) { - var MAX_RETRIES = 5; - var attemptedRetries = 0; - var error = new Error('Rate Limit Error.'); - error.code = 429; // Rate limit error - var authorizedReqOpts = { a: 'b', c: 'd' }; + it('should execute callback with authorization error', function(done) { + var error = new Error('Error.'); - var old_setTimeout = setTimeout; - setTimeout = function(callback, time) { - var MIN_TIME = (Math.pow(2, attemptedRetries) * 1000); - var MAX_TIME = (Math.pow(2, attemptedRetries) * 1000) + 1000; - assert(time >= MIN_TIME && time <= MAX_TIME); - attemptedRetries++; - callback(); // make the request again + fakeAuthClient.authorize = function(cb) { + cb(error); }; - gsa_Override = function() { - return function authorize(reqOpts, callback) { - callback(null, authorizedReqOpts); - }; + utilOverrides.getAuthClient = function(config, callback) { + callback(null, fakeAuthClient); }; - request_Override = function(reqOpts, callback) { - callback(error); // this callback should check for rate limits - }; + var makeAuthorizedRequest = + util.makeAuthorizedRequestFactory(config, assert.ifError); - var makeRequest = util.makeAuthorizedRequest({ - maxRetries: MAX_RETRIES - }); - - makeRequest({}, function(err) { - setTimeout = old_setTimeout; - assert.equal(attemptedRetries, MAX_RETRIES); - assert.equal(err, error); + makeAuthorizedRequest.getCredentials(function(err) { + assert.deepEqual(err, error); done(); }); }); - it('should not retry rate limits if autoRetry is false', function(done) { - var attemptedRetries = 0; - var error = new Error('Rate Limit Error.'); - error.code = 429; // Rate limit error - - var authorizedReqOpts = { a: 'b', c: 'd' }; - - var old_setTimeout = setTimeout; - setTimeout = function(callback, time) { - var MIN_TIME = (Math.pow(2, attemptedRetries) * 1000); - var MAX_TIME = (Math.pow(2, attemptedRetries) * 1000) + 1000; - assert(time >= MIN_TIME && time <= MAX_TIME); - attemptedRetries++; - callback(); // make the request again + it('should exec callback with client_email & client_key', function(done) { + fakeAuthClient.authorize = function(callback) { + callback(); }; - gsa_Override = function() { - return function authorize(reqOpts, callback) { - callback(null, authorizedReqOpts); - }; + utilOverrides.getAuthClient = function(config, callback) { + callback(null, fakeAuthClient); }; - request_Override = function(reqOpts, callback) { - callback(error); // this callback should check for rate limits - }; + var makeAuthorizedRequest = + util.makeAuthorizedRequestFactory(config, assert.ifError); - var makeRequest = util.makeAuthorizedRequest({ - autoRetry: false - }); + makeAuthorizedRequest.getCredentials(function(err, credentials) { + assert.deepEqual(credentials, { + client_email: fakeAuthClient.email, + private_key: fakeAuthClient.key + }); - makeRequest({}, function(err) { - setTimeout = old_setTimeout; - assert.equal(attemptedRetries, 0); - assert.equal(err, error); done(); }); }); @@ -778,4 +758,227 @@ describe('common/util', function() { assert.equal(obj.prop, 'value'); }); }); + + describe('shouldRetryRequest', function() { + it('should return false if there is no error', function() { + assert.strictEqual(util.shouldRetryRequest(), false); + }); + + it('should return false from generic error', function() { + var error = new Error('Generic error with no code'); + + assert.strictEqual(util.shouldRetryRequest(error), false); + }); + + it('should return true with error code 429', function() { + var error = new Error('429'); + error.code = 429; + + assert.strictEqual(util.shouldRetryRequest(error), true); + }); + + it('should return true with error code 500', function() { + var error = new Error('500'); + error.code = 500; + + assert.strictEqual(util.shouldRetryRequest(error), true); + }); + + it('should return true with error code 503', function() { + var error = new Error('503'); + error.code = 503; + + assert.strictEqual(util.shouldRetryRequest(error), true); + }); + + it('should detect rateLimitExceeded reason', function() { + var rateLimitError = new Error('Rate limit error without code.'); + rateLimitError.errors = [{ reason: 'rateLimitExceeded' }]; + + assert.strictEqual(util.shouldRetryRequest(rateLimitError), true); + }); + + it('should detect userRateLimitExceeded reason', function() { + var rateLimitError = new Error('Rate limit error without code.'); + rateLimitError.errors = [{ reason: 'userRateLimitExceeded' }]; + + assert.strictEqual(util.shouldRetryRequest(rateLimitError), true); + }); + }); + + describe('getNextRetryWait', function() { + function secs(seconds) { + return seconds * 1000; + } + + it('should return exponential retry delay', function() { + [1, 2, 3, 4, 5].forEach(assertTime); + + function assertTime(retryNumber) { + var min = (Math.pow(2, retryNumber) * secs(1)); + var max = (Math.pow(2, retryNumber) * secs(1)) + secs(1); + + var time = util.getNextRetryWait(retryNumber); + + assert(time >= min && time <= max); + } + }); + }); + + describe('makeRequest', function() { + var PKG = require('../../package.json'); + var USER_AGENT = 'gcloud-node/' + PKG.version; + var reqOpts = { a: 'b', c: 'd' }; + var expectedReqOpts = extend(true, {}, reqOpts, { + headers: { + 'User-Agent': USER_AGENT + } + }); + + it('should make a request', function(done) { + request_Override = function() { + done(); + }; + + util.makeRequest({}, assert.ifError, {}); + }); + + it('should add the user agent', function(done) { + request_Override = function(rOpts) { + assert.deepEqual(rOpts, expectedReqOpts); + done(); + }; + + util.makeRequest(reqOpts, assert.ifError, {}); + }); + + it('should let handleResp handle the response', function(done) { + var error = new Error('Error.'); + var response = { a: 'b', c: 'd' }; + var body = response.a; + + request_Override = function(rOpts, callback) { + callback(error, response, body); + }; + + utilOverrides.handleResp = function(err, resp, bdy) { + assert.deepEqual(err, error); + assert.deepEqual(resp, response); + assert.deepEqual(bdy, body); + done(); + }; + + util.makeRequest({}, {}, assert.ifError); + }); + + describe('request errors', function() { + describe('non-rate limit error', function() { + it('should return error to callback', function(done) { + var nonRateLimitError = new Error('Error.'); + + request_Override = function(reqOpts, callback) { + callback(nonRateLimitError); + }; + + util.makeRequest({}, {}, function(err) { + assert.deepEqual(err, nonRateLimitError); + done(); + }); + }); + }); + + describe('rate limit errors', function() { + var rateLimitError = new Error('Rate limit error.'); + rateLimitError.code = 500; + + beforeEach(function() { + // Always return a rate limit error. + request_Override = function (reqOpts, callback) { + callback(rateLimitError); + }; + + // Always suggest retrying. + utilOverrides.shouldRetryRequest = function() { + return true; + }; + + // Always return a 0 retry wait. + utilOverrides.getNextRetryWait = function() { + return 0; + }; + }); + + it('should check with shouldRetryRequest', function(done) { + utilOverrides.shouldRetryRequest = function() { + done(); + }; + + util.makeRequest({}, {}, util.noop); + }); + + it('should default to 3 retries', function(done) { + var attempts = 0; + var expectedAttempts = 4; // the original request + 3 retries + + utilOverrides.handleResp = function(err, resp, body, callback) { + attempts++; + callback(err); + }; + + util.makeRequest({}, {}, function(err) { + assert.equal(err, rateLimitError); + assert.equal(attempts, expectedAttempts); + done(); + }); + }); + + it('should allow max retries to be specified', function(done) { + var attempts = 0; + var maxRetries = 5; + var expectedAttempts = maxRetries + 1; // the original req + + utilOverrides.handleResp = function(err, resp, body, callback) { + attempts++; + callback(err); + }; + + util.makeRequest({}, { maxRetries: maxRetries }, function(err) { + assert.equal(err, rateLimitError); + assert.equal(attempts, expectedAttempts); + done(); + }); + }); + + it('should not retry reqs if autoRetry is false', function(done) { + var attempts = 0; + var expectedAttempts = 1; // the original req + + utilOverrides.handleResp = function(err, resp, body, callback) { + attempts++; + callback(err); + }; + + util.makeRequest({}, { autoRetry: false }, function(err) { + assert.equal(err, rateLimitError); + assert.equal(attempts, expectedAttempts); + done(); + }); + }); + }); + }); + + describe('request success', function() { + it('should let handleResp handle response', function(done) { + utilOverrides.handleResp = function() { + done(); + }; + + request_Override = function(reqOpts, callback) { + callback(); + }; + + util.makeRequest({}, {}, assert.ifError); + }); + }); + }); }); diff --git a/test/pubsub/index.js b/test/pubsub/index.js index f613c6f22a0..d52c69b1f81 100644 --- a/test/pubsub/index.js +++ b/test/pubsub/index.js @@ -202,11 +202,15 @@ describe('PubSub', function() { }); }); - it('should pass network requests to the connection object', function(done) { - var pubsub = new PubSub(); - request_Override = function() { - done(); - }; - pubsub.makeReq_(); + describe('makeReq_', function() { + it('should pass network requests to the connection object', function(done) { + var pubsub = new PubSub(); + + pubsub.makeAuthorizedRequest_ = function() { + done(); + }; + + pubsub.makeReq_(null, null, null, null, assert.ifError); + }); }); });