BigQuery samples.

bigquery/README.md

@@ -0,0 +1,25 @@
+## BigQuery Samples
+
+These samples require two environment variables to be set:
+
+- `GOOGLE_APPLICATION_CREDENTIALS` - Path to a service account file. You can
+download one from your Google project's "permissions" page.
+- `GCLOUD_PROJECT` - Id of your Google project.
+
+## Run a sample
+
+Install dependencies:
+
+    npm install
+
+To print available commands:
+
+    npm run
+
+Execute a sample:
+
+    npm run <sample>
+
+Example:
+
+    npm run dataset_size -- bigquery-public-data hacker_news

bigquery/dataset_size.js

@@ -0,0 +1,151 @@
+// Copyright 2016, Google, Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+'use strict';
+
+var async = require('async');
+
+// [START auth]
+// You must set the GOOGLE_APPLICATION_CREDENTIALS and GCLOUD_PROJECT
+// environment variables to run this sample
+var projectId = process.env.GCLOUD_PROJECT;
+
+// Initialize gcloud
+var gcloud = require('gcloud')({
+  projectId: projectId
+});
+
+// Get a reference to the bigquery component
+var bigquery = gcloud.bigquery();
+// [END auth]
+
+// [START list_tables]
+/**
+ * Retrieve all tables for the specified dataset.
+ *
+ * @param {Object} bigquery gcloud-node bigquery client.
+ * @param {string} datasetId Dataset of the tables to retrieve.
+ * @param {string} [pageToken] Page to retrieve.
+ * @param {Function} callback Callback function.
+ */
+function getAllTablesExample(bigquery, datasetId, pageToken, callback) {
+  if (typeof pageToken === 'function') {
+    callback = pageToken;
+    pageToken = undefined;
+  }
+  var dataset = bigquery.dataset(datasetId);
+  var options = {};
+  if (pageToken) {
+    options.pageToken = pageToken;
+  }
+
+  // Grab paginated tables
+  dataset.getTables(options, function (err, tables, nextQuery) {
+    // Quit on error
+    if (err) {
+      return callback(err);
+    }
+
+    // There is another page of tables
+    if (nextQuery) {
+      // Grab the remaining pages of tables recursively
+      return getAllTablesExample(datasetId, nextQuery.token, function (err, _tables) {
+        if (_tables) {
+          tables = tables.concat(_tables);
+        }
+        callback(null, tables);
+      });
+    }
+    // Last page of tables
+    return callback(null, tables);
+  });
+}
+// [END list_tables]
+
+// [START get_size]
+/**
+ * Retrieve the size of the specified dataset.
+ *
+ * @param {string} projectId The project, .e.g. "bigquery-public-data"
+ * @param {string} datasetId The dataset, e.g. "hacker_news"
+ * @param {Function} callback Callback function.
+ */
+function getSizeExample(projectId, datasetId, callback) {
+  if (!projectId) {
+    return callback(new Error('projectId is required!'));
+  }
+  if (!datasetId) {
+    return callback(new Error('datasetId is require!'));
+  }
+
+  var gcloud = require('gcloud')({
+    projectId: projectId || process.env.GCLOUD_PROJECT
+  });
+  var bigquery = gcloud.bigquery();
+
+  // Fetch all tables in the dataset
+  getAllTablesExample(bigquery, datasetId, function (err, tables) {
+    return async.parallel(tables.map(function (table) {
+      return function (cb) {
+        // Fetch more detailed info for each table
+        table.get(function (err, tableInfo) {
+          if (err) {
+            return cb(err);
+          }
+          // Return numBytes converted to Megabytes
+          return cb(null, (parseInt(table.metadata.numBytes, 10) / 1000) / 1000);
+        });
+      };
+    }), function (err, sizes) {
+      if (err) {
+        return callback(err);
+      }
+      var sum = sizes.reduce(function (cur, prev) {
+        return cur + prev;
+      }, 0);
+      return callback(null, sum);
+    });
+  });
+}
+// [END get_size]
+
+// Run the examples
+exports.main = function (projectId, datasetId, cb) {
+  getSizeExample(projectId, datasetId, function (err, sum) {
+    if (err) {
+      return cb(err);
+    }
+    var size = 'MB';
+    if (sum > 1000) {
+      sum = sum / 1000;
+      size = 'GB';
+    }
+    if (sum > 1000) {
+      sum = sum / 1000;
+      size = 'TB';
+    }
+    cb(null, '' + sum.toPrecision(5) + ' ' + size);
+  });
+};
+
+if (module === require.main) {
+  var args = process.argv.slice(2);
+  if (args.length !== 2) {
+    throw new Error('Usage: node dataset_size.js <projectId> <datasetId>');
+  }
+  exports.main(
+    args[0],
+    args[1],
+    console.log
+  );
+}

bigquery/getting_started.js

@@ -0,0 +1,79 @@
+// Copyright 2016, Google, Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// [START complete]
+'use strict';
+
+// [START auth]
+// You must set the GOOGLE_APPLICATION_CREDENTIALS and GCLOUD_PROJECT
+// environment variables to run this sample
+var projectId = process.env.GCLOUD_PROJECT;
+
+// Initialize gcloud
+var gcloud = require('gcloud')({
+  projectId: projectId
+});
+
+// Get a reference to the bigquery component
+var bigquery = gcloud.bigquery();
+// [END auth]
+
+// [START print]
+function printExample(rows) {
+  console.log('Query Results:');
+  rows.forEach(function (row) {
+    var str = '';
+    for (var key in row) {
+      if (str) {
+        str += '\t';
+      }
+      str += key + ': ' + row[key];
+    }
+    console.log(str);
+  });
+}
+// [END print]
+
+// [START query]
+/**
+ * Run an example query.
+ *
+ * @param {Function} callback Callback function.
+ */
+function queryExample(callback) {
+  var query = 'SELECT TOP(corpus, 10) as title, COUNT(*) as unique_words\n' +
+    'FROM [publicdata:samples.shakespeare];'
+
+  bigquery.query(query, function(err, rows) {
+    if (err) {
+      return callback(err);
+    }
+
+    printExample(rows);
+    callback(null, rows);
+  });
+}
+// [END query]
+
+// [END complete]
+
+// Run the examples
+exports.main = function (cb) {
+  queryExample(cb);
+};
+
+if (module === require.main) {
+  exports.main(
+    console.log
+  );
+}

bigquery/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "nodejs-docs-samples-bigquery",
+  "description": "Node.js samples for Google BigQuery.",
+  "version": "0.0.1",
+  "private": true,
+  "license": "Apache Version 2.0",
+  "author": "Google Inc.",
+  "engines": {
+    "node": ">=0.10.x"
+  },
+  "scripts": {
+    "dataset_size": "node dataset_size.js"
+  },
+  "dependencies": {
+    "async": "^1.5.2",
+    "gcloud": "^0.29.0"
+  }
+}

package.json

@@ -26,10 +26,12 @@
   "scripts": {
     "jshint": "jshint --exclude-path=.jshintignore .",
     "deps_appengine": "ava --match='*: dependencies should install*'",
-    "ava": "npm run deps_appengine && ava --match='!*: dependencies should install*'",
+    "ava": "ava --match='!*: dependencies should install*'",
+    "ava:deps": "npm run deps_appengine && npm run ava",
     "cover": "npm run deps_appengine && nyc ava --match='!*: dependencies should install*'",
     "report": "nyc report --reporter=text-lcov | ./node_modules/.bin/coveralls",
     "report-html": "nyc report --reporter=html",
+    "deps_bigquery": "cd bigquery; npm i; cd ../",
     "deps_datastore": "cd datastore; npm i; cd ../",
     "deps_pubsub": "cd pubsub; npm i; cd ../",
     "deps_monitoring": "cd monitoring; npm i; cd ../",
@@ -39,7 +41,7 @@
     "deps_functions": "cd functions/uuid; npm i; cd ../..",
     "deps_sendgrid": "cd computeengine/sendgrid; npm i; cd ../..",
     "pretest_geddy": "cd appengine/geddy; npm i geddy; GEDDY_SECRET=config/secrets.json; [[ -f $GEDDY_SECRET ]] || echo '{}' > $GEDDY_SECRET && node node_modules/.bin/geddy gen secret; cd ../..;",
-    "pretest": "npm run deps_datastore; npm run deps_monitoring; npm run deps_storage; npm run deps_pubsub; npm run deps_prediction; npm run deps_logging; npm run deps_functions; npm run deps_sendgrid; npm run pretest_geddy",
+    "pretest": "npm run deps_bigquery; npm run deps_datastore; npm run deps_monitoring; npm run deps_storage; npm run deps_pubsub; npm run deps_prediction; npm run deps_logging; npm run deps_functions; npm run deps_sendgrid; npm run pretest_geddy",
     "test": "npm run jshint && npm run cover"
   },
   "ava": {

pubsub/subscription.js

@@ -20,7 +20,6 @@ var async = require('async');
 // environment variables to run this sample
 var projectId = process.env.GCLOUD_PROJECT;
 
-// [START require]
 // Initialize gcloud
 var gcloud = require('gcloud')({
   projectId: projectId

test/bigquery/dataset_size.test.js

@@ -0,0 +1,30 @@
+// Copyright 2016, Google, Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+'use strict';
+
+var test = require('ava');
+var datasetSizeExample = require('../../bigquery/dataset_size');
+
+test.cb('should return the size of a dataset', function (t) {
+  datasetSizeExample.main(
+    'bigquery-public-data',
+    'hacker_news',
+    function (err, size) {
+      t.ifError(err);
+      t.is(typeof size, 'string');
+      t.ok(size.indexOf(' GB') === size.length - 3);
+      t.end();
+    }
+  );
+});

test/bigquery/getting_started.test.js

@@ -0,0 +1,28 @@
+// Copyright 2016, Google, Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+'use strict';
+
+var test = require('ava');
+var gettingStartedExample = require('../../bigquery/getting_started');
+
+test.cb('should run a query', function (t) {
+  gettingStartedExample.main(
+    function (err, rows) {
+      t.ifError(err);
+      t.ok(Array.isArray(rows));
+      t.is(rows.length, 10);
+      t.end();
+    }
+  );
+});