From 78f0b78b389a02d5d2306efffc0334f70f5d099f Mon Sep 17 00:00:00 2001 From: Stuart Paterson Date: Mon, 16 Apr 2018 10:12:06 +0100 Subject: [PATCH 1/7] Initial import of transport for GCP. Signed-off-by: Stuart Paterson --- .../platforms/detect/specifications/api.rb | 1 + lib/train/transports/gcp.rb | 101 +++++++++ test/unit/transports/gcp_test.rb | 191 ++++++++++++++++++ train.gemspec | 3 + 4 files changed, 296 insertions(+) create mode 100644 lib/train/transports/gcp.rb create mode 100644 test/unit/transports/gcp_test.rb diff --git a/lib/train/platforms/detect/specifications/api.rb b/lib/train/platforms/detect/specifications/api.rb index d13ca6fb..b7dbeb93 100644 --- a/lib/train/platforms/detect/specifications/api.rb +++ b/lib/train/platforms/detect/specifications/api.rb @@ -10,6 +10,7 @@ def self.load plat.family('cloud').in_family('api') plat.name('aws').in_family('cloud') plat.name('azure').in_family('cloud') + plat.name('gcp').in_family('cloud') end end end diff --git a/lib/train/transports/gcp.rb b/lib/train/transports/gcp.rb new file mode 100644 index 00000000..23efacce --- /dev/null +++ b/lib/train/transports/gcp.rb @@ -0,0 +1,101 @@ +# encoding: utf-8 + +require 'train/plugins' +require 'google/apis' +require 'google/apis/cloudresourcemanager_v1' +require 'google/apis/compute_v1' +require 'google/apis/storage_v1' +require 'google/apis/iam_v1' +require 'googleauth' +require 'JSON' + +module Train::Transports + class Gcp < Train.plugin(1) + name 'gcp' + + # GCP will look automatically for the below env var for service accounts etc. : + option :google_application_credentials, required: false, default: ENV['GOOGLE_APPLICATION_CREDENTIALS'] + # see https://cloud.google.com/docs/authentication/production#providing_credentials_to_your_application + # In the absence of this, the client is expected to have already set up local credentials via: + # $ gcloud auth application-default login + # $ gcloud config set project + # GCP projects can have default regions / zones set, see: + # https://cloud.google.com/compute/docs/regions-zones/changing-default-zone-region + # can also specify project via env var: + option :google_cloud_project, required: false, default: ENV['GOOGLE_CLOUD_PROJECT'] + + def connection(_ = nil) + @connection ||= Connection.new(@options) + end + + class Connection < BaseConnection + def initialize(options) + super(options) + + # additional GCP platform metadata + release = Gem.loaded_specs['google_cloud'] + @platform_details = { release: "google-cloud-v#{release}" } + + # Initialize the client object cache + @cache_enabled[:api_call] = true + @cache[:api_call] = {} + + connect + end + + def platform + direct_platform('gcp', @platform_details) + end + + # Instantiate some named classes for ease of use + def gcp_compute_client + klass = Google::Apis::ComputeV1::ComputeService + return klass.new unless cache_enabled?(:api_call) + @cache[:api_call][klass.to_s.to_sym] ||= klass.new + end + + def gcp_iam_client + klass = Google::Apis::IamV1::IamService + return klass.new unless cache_enabled?(:api_call) + @cache[:api_call][klass.to_s.to_sym] ||= klass.new + end + + def gcp_project_client + klass = Google::Apis::CloudresourcemanagerV1::CloudResourceManagerService + return klass.new unless cache_enabled?(:api_call) + @cache[:api_call][klass.to_s.to_sym] ||= klass.new + end + + def gcp_storage_client + klass = Google::Apis::StorageV1::StorageService + return klass.new unless cache_enabled?(:api_call) + @cache[:api_call][klass.to_s.to_sym] ||= klass.new + end + + # Let's allow for other clients too + def gcp_client(klass) + return klass.new unless cache_enabled?(:api_call) + @cache[:api_call][klass.to_s.to_sym] ||= klass.new + end + + def connect + ENV['GOOGLE_APPLICATION_CREDENTIALS'] = @options[:google_application_credentials] if @options[:google_application_credentials] + ENV['GOOGLE_CLOUD_PROJECT'] = @options[:google_cloud_project] if @options[:google_cloud_project] + # GCP initialization + scopes = ['https://www.googleapis.com/auth/cloud-platform', + 'https://www.googleapis.com/auth/compute'] + authorization = Google::Auth.get_application_default(scopes) + Google::Apis::RequestOptions.default.authorization = authorization + end + + def uri + "gcp://#{unique_identifier}" + end + + def unique_identifier + # use auth client_id - same to retrieve for any of the clients but use IAM + gcp_iam_client.request_options.authorization.client_id + end + end + end +end diff --git a/test/unit/transports/gcp_test.rb b/test/unit/transports/gcp_test.rb new file mode 100644 index 00000000..829de81d --- /dev/null +++ b/test/unit/transports/gcp_test.rb @@ -0,0 +1,191 @@ +# encoding: utf-8 + +require 'helper' + +describe 'gcp transport' do + + let(:credentials_file) do + require 'tempfile' + file = Tempfile.new('application_default_credentials.json') + info = <<-INFO +{ + "client_id": "asdfasf-asdfasdf.apps.googleusercontent.com", + "client_secret": "d-asdfasdf", + "refresh_token": "1/adsfasdf-lCkju3-yQmjr20xVZonrfkE48L", + "type": "authorized_user" +} + INFO + file.write(info) + file.close + file + end + + let(:credentials_file_override) do + require 'tempfile' + file = Tempfile.new('application_default_credentials.json') + info = <<-INFO +{ + "client_id": "asdfasf-asdfasdf.apps.googleusercontent.com", + "client_secret": "d-asdfasdf", + "refresh_token": "1/adsfasdf-lCkju3-yQmjr20xVZonrfkE48L", + "type": "authorized_user" +} + INFO + file.write(info) + file.close + file + end + + def transport(options = nil) + ENV['GOOGLE_APPLICATION_CREDENTIALS'] = credentials_file.path + ENV['GOOGLE_CLOUD_PROJECT'] = 'test_project' + # need to require this at here as it captures the envs on load + require 'train/transports/gcp' + Train::Transports::Gcp.new(options) + end + + let(:connection) { transport.connection } + let(:options) { connection.instance_variable_get(:@options) } + let(:cache) { connection.instance_variable_get(:@cache) } + + describe 'options' do + it 'defaults to env options' do + options[:google_application_credentials] = credentials_file.path + options[:google_cloud_project].must_equal 'test_project' + end + end + + it 'allows for options override' do + transport = transport(google_application_credentials: credentials_file_override.path, google_cloud_project: "override_project") + options = transport.connection.instance_variable_get(:@options) + options[:google_application_credentials].must_equal credentials_file_override.path + options[:google_cloud_project].must_equal "override_project" + end + + describe 'platform' do + it 'returns platform' do + platform = connection.platform + platform.name.must_equal 'gcp' + platform.family_hierarchy.must_equal ['cloud', 'api'] + end + end + + describe 'gcp_client' do + it 'test gcp_client with caching' do + client = connection.gcp_client(Object) + client.is_a?(Object).must_equal true + cache[:api_call].count.must_equal 1 + end + + it 'test gcp_client without caching' do + connection.disable_cache(:api_call) + client = connection.gcp_client(Object) + client.is_a?(Object).must_equal true + cache[:api_call].count.must_equal 0 + end + end + + + describe 'gcp_compute_client' do + it 'test gcp_compute_client with caching' do + client = connection.gcp_compute_client + client.is_a?(Google::Apis::ComputeV1::ComputeService).must_equal true + cache[:api_call].count.must_equal 1 + end + + it 'test gcp_client without caching' do + connection.disable_cache(:api_call) + client = connection.gcp_compute_client + client.is_a?(Google::Apis::ComputeV1::ComputeService).must_equal true + cache[:api_call].count.must_equal 0 + end + end + + describe 'gcp_iam_client' do + it 'test gcp_iam_client with caching' do + client = connection.gcp_iam_client + client.is_a?(Google::Apis::IamV1::IamService).must_equal true + cache[:api_call].count.must_equal 1 + end + + it 'test gcp_iam_client without caching' do + connection.disable_cache(:api_call) + client = connection.gcp_iam_client + client.is_a?(Google::Apis::IamV1::IamService).must_equal true + cache[:api_call].count.must_equal 0 + end + end + + describe 'gcp_project_client' do + it 'test gcp_project_client with caching' do + client = connection.gcp_project_client + client.is_a?(Google::Apis::CloudresourcemanagerV1::CloudResourceManagerService).must_equal true + cache[:api_call].count.must_equal 1 + end + + it 'test gcp_project_client without caching' do + connection.disable_cache(:api_call) + client = connection.gcp_project_client + client.is_a?(Google::Apis::CloudresourcemanagerV1::CloudResourceManagerService).must_equal true + cache[:api_call].count.must_equal 0 + end + end + + describe 'gcp_storage_client' do + it 'test gcp_storage_client with caching' do + client = connection.gcp_storage_client + client.is_a?(Google::Apis::StorageV1::StorageService).must_equal true + cache[:api_call].count.must_equal 1 + end + + it 'test gcp_storage_client without caching' do + connection.disable_cache(:api_call) + client = connection.gcp_storage_client + client.is_a?(Google::Apis::StorageV1::StorageService).must_equal true + cache[:api_call].count.must_equal 0 + end + end + + # test options override of env vars in connect + describe 'connect' do + let(:creds) do + require 'tempfile' + file = Tempfile.new('creds') + info = <<-INFO +{ + "client_id": "asdfasf-asdfasdf.apps.googleusercontent.com", + "client_secret": "d-asdfasdf", + "refresh_token": "1/adsfasdf-lCkju3-yQmjr20xVZonrfkE48L", + "type": "authorized_user" +} + INFO + file.write(info) + file.close + file + end + it 'validate gcp connection with credentials' do + options[:google_application_credentials] = creds.path + connection.connect + ENV['GOOGLE_APPLICATION_CREDENTIALS'].must_equal creds.path + end + it 'validate gcp connection with project' do + options[:google_cloud_project] = 'project' + connection.connect + ENV['GOOGLE_CLOUD_PROJECT'].must_equal 'project' + end + end + + describe 'unique_identifier' do + it 'test connection unique identifier' do + client = connection + client.unique_identifier.must_equal 'asdfasf-asdfasdf.apps.googleusercontent.com' + end + end + + describe 'uri' do + it 'test uri' do + client = connection + client.uri.must_equal 'gcp://asdfasf-asdfasdf.apps.googleusercontent.com' + end + end +end \ No newline at end of file diff --git a/train.gemspec b/train.gemspec index 68fe14a7..0618e9c4 100644 --- a/train.gemspec +++ b/train.gemspec @@ -35,6 +35,9 @@ Gem::Specification.new do |spec| spec.add_dependency 'docker-api', '~> 1.26' spec.add_dependency 'aws-sdk', '~> 2' spec.add_dependency 'azure_mgmt_resources', '~> 0.15' + spec.add_dependency 'google-api-client', '~> 0.19.8' + spec.add_dependency 'googleauth', '~> 0.6.2' + spec.add_dependency 'google-cloud', '~> 0.51.1' spec.add_dependency 'inifile' spec.add_development_dependency 'mocha', '~> 1.1' From 368d4fdae1dd1f59f2e92fd416ca8c736592f37c Mon Sep 17 00:00:00 2001 From: Stuart Paterson Date: Wed, 18 Apr 2018 09:25:58 +0100 Subject: [PATCH 2/7] Updating named clients to use gcp_client to avoid duplication Signed-off-by: Stuart Paterson --- lib/train/transports/gcp.rb | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/lib/train/transports/gcp.rb b/lib/train/transports/gcp.rb index 23efacce..23c560af 100644 --- a/lib/train/transports/gcp.rb +++ b/lib/train/transports/gcp.rb @@ -49,27 +49,19 @@ def platform # Instantiate some named classes for ease of use def gcp_compute_client - klass = Google::Apis::ComputeV1::ComputeService - return klass.new unless cache_enabled?(:api_call) - @cache[:api_call][klass.to_s.to_sym] ||= klass.new + gcp_client(Google::Apis::ComputeV1::ComputeService) end def gcp_iam_client - klass = Google::Apis::IamV1::IamService - return klass.new unless cache_enabled?(:api_call) - @cache[:api_call][klass.to_s.to_sym] ||= klass.new + gcp_client(Google::Apis::IamV1::IamService) end def gcp_project_client - klass = Google::Apis::CloudresourcemanagerV1::CloudResourceManagerService - return klass.new unless cache_enabled?(:api_call) - @cache[:api_call][klass.to_s.to_sym] ||= klass.new + gcp_client(Google::Apis::CloudresourcemanagerV1::CloudResourceManagerService) end def gcp_storage_client - klass = Google::Apis::StorageV1::StorageService - return klass.new unless cache_enabled?(:api_call) - @cache[:api_call][klass.to_s.to_sym] ||= klass.new + gcp_client(Google::Apis::StorageV1::StorageService) end # Let's allow for other clients too From 0a3a587490db28538223f8c65ef6fabb6a69ac64 Mon Sep 17 00:00:00 2001 From: Stuart Paterson Date: Wed, 18 Apr 2018 09:36:03 +0100 Subject: [PATCH 3/7] Updating to latest berkshelf version to avoid conflicts with google dependencies. Signed-off-by: Stuart Paterson --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index b5db3d3e..0a61c400 100644 --- a/Gemfile +++ b/Gemfile @@ -23,7 +23,7 @@ group :test do end group :integration do - gem 'berkshelf', '~> 4.3' + gem 'berkshelf', '~> 6.3.2' gem 'test-kitchen', '~> 1.11' gem 'kitchen-vagrant' end From 9986fbe87dcb880766b2ebc6934d84ffcd551900 Mon Sep 17 00:00:00 2001 From: Stuart Paterson Date: Wed, 18 Apr 2018 10:03:40 +0100 Subject: [PATCH 4/7] Removing unnecessary JSON import. Signed-off-by: Stuart Paterson --- lib/train/transports/gcp.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/train/transports/gcp.rb b/lib/train/transports/gcp.rb index 23c560af..4c27ccd2 100644 --- a/lib/train/transports/gcp.rb +++ b/lib/train/transports/gcp.rb @@ -7,7 +7,6 @@ require 'google/apis/storage_v1' require 'google/apis/iam_v1' require 'googleauth' -require 'JSON' module Train::Transports class Gcp < Train.plugin(1) From 0019eef7e6f1333941ab2999649e6004729d2454 Mon Sep 17 00:00:00 2001 From: Stuart Paterson Date: Tue, 24 Apr 2018 14:58:57 +0100 Subject: [PATCH 5/7] Pinned google-protobuf version to avoid build issue. Signed-off-by: Stuart Paterson --- train.gemspec | 1 + 1 file changed, 1 insertion(+) diff --git a/train.gemspec b/train.gemspec index 0618e9c4..dc834d72 100644 --- a/train.gemspec +++ b/train.gemspec @@ -38,6 +38,7 @@ Gem::Specification.new do |spec| spec.add_dependency 'google-api-client', '~> 0.19.8' spec.add_dependency 'googleauth', '~> 0.6.2' spec.add_dependency 'google-cloud', '~> 0.51.1' + spec.add_dependency 'google-protobuf','= 3.5.1' spec.add_dependency 'inifile' spec.add_development_dependency 'mocha', '~> 1.1' From a60564d142861271e424843fba4d19dfd533be08 Mon Sep 17 00:00:00 2001 From: Stuart Paterson Date: Tue, 24 Apr 2018 15:47:24 +0100 Subject: [PATCH 6/7] Remove unnecessary space. Signed-off-by: Stuart Paterson --- train.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train.gemspec b/train.gemspec index dc834d72..d3de214c 100644 --- a/train.gemspec +++ b/train.gemspec @@ -38,7 +38,7 @@ Gem::Specification.new do |spec| spec.add_dependency 'google-api-client', '~> 0.19.8' spec.add_dependency 'googleauth', '~> 0.6.2' spec.add_dependency 'google-cloud', '~> 0.51.1' - spec.add_dependency 'google-protobuf','= 3.5.1' + spec.add_dependency 'google-protobuf', '= 3.5.1' spec.add_dependency 'inifile' spec.add_development_dependency 'mocha', '~> 1.1' From 9cadb9ffea393b5046aa1464cf2cedd4d11dc875 Mon Sep 17 00:00:00 2001 From: Stuart Paterson Date: Wed, 25 Apr 2018 09:10:27 +0100 Subject: [PATCH 7/7] Downgrading berkshelf to help with integration testing. Signed-off-by: Stuart Paterson --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index 0a61c400..10837d8f 100644 --- a/Gemfile +++ b/Gemfile @@ -23,7 +23,7 @@ group :test do end group :integration do - gem 'berkshelf', '~> 6.3.2' + gem 'berkshelf', '~> 5.2' gem 'test-kitchen', '~> 1.11' gem 'kitchen-vagrant' end