From 6af1ed8819b006a69281b822e92dc03723b90540 Mon Sep 17 00:00:00 2001 From: danielsdeleo Date: Mon, 10 Aug 2015 19:05:59 -0700 Subject: [PATCH] Add service class to GC cookbook_artifacts --- .../clean_policy_cookbooks.rb | 116 ++++++++ lib/chef-dk/service_exceptions.rb | 3 + .../clean_policy_cookbooks_spec.rb | 261 ++++++++++++++++++ 3 files changed, 380 insertions(+) create mode 100644 lib/chef-dk/policyfile_services/clean_policy_cookbooks.rb create mode 100644 spec/unit/policyfile_services/clean_policy_cookbooks_spec.rb diff --git a/lib/chef-dk/policyfile_services/clean_policy_cookbooks.rb b/lib/chef-dk/policyfile_services/clean_policy_cookbooks.rb new file mode 100644 index 000000000..282021757 --- /dev/null +++ b/lib/chef-dk/policyfile_services/clean_policy_cookbooks.rb @@ -0,0 +1,116 @@ +# +# Copyright:: Copyright (c) 2015 Chef Software Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'set' + +require 'chef-dk/authenticated_http' +require 'chef-dk/service_exceptions' + +module ChefDK + module PolicyfileServices + + class CleanPolicyCookbooks + + attr_reader :chef_config + + def initialize(config: nil, ui: nil) + @chef_config = config + @ui = ui + + @all_cookbooks = nil + @active_cookbooks = nil + @all_policies = nil + end + + def run + gc_cookbooks + rescue => e + raise PolicyCookbookCleanError.new("Failed to cleanup policy cookbooks", e) + end + + def gc_cookbooks + cookbooks_to_clean.each do |name, identifiers| + identifiers.each do |identifier| + http_client.delete("/cookbook_artifacts/#{name}/#{identifier}") + end + end + end + + + def all_cookbooks + cookbook_list = http_client.get("/cookbook_artifacts") + cookbook_list.inject({}) do |cb_map, (name, cb_info)| + cb_map[name] = cb_info["versions"].map { |v| v["identifier"] } + cb_map + end + end + + def active_cookbooks + policy_revisions_by_name.inject({}) do |cb_map, (policy_name, revision_ids)| + revision_ids.each do |revision_id| + cookbook_revisions_in_policy(policy_name, revision_id).each do |cb_name, identifier| + cb_map[cb_name] ||= Set.new + cb_map[cb_name] << identifier + end + end + cb_map + end + end + + def cookbooks_to_clean + active_cbs = active_cookbooks + + all_cookbooks.inject({}) do |cb_map, (cb_name, revisions)| + active_revs = active_cbs[cb_name] + inactive_revs = Set.new(revisions) - active_revs + cb_map[cb_name] = inactive_revs unless inactive_revs.empty? + + cb_map + end + end + + # @api private + def policy_revisions_by_name + policies_list = http_client.get("/policies") + policies_list.inject({}) do |policies_map, (name, policy_info)| + policies_map[name] = policy_info["revisions"].keys + policies_map + end + end + + # @api private + def cookbook_revisions_in_policy(name, revision_id) + policy_revision_data = http_client.get("/policies/#{name}/revisions/#{revision_id}") + + policy_revision_data["cookbook_locks"].inject({}) do |cb_map, (cb_name, lock_info)| + cb_map[cb_name] = lock_info["identifier"] + cb_map + end + end + + # @api private + # An instance of ChefDK::AuthenticatedHTTP configured with the user's + # server URL and credentials. + def http_client + @http_client ||= ChefDK::AuthenticatedHTTP.new(chef_config.chef_server_url, + signing_key_filename: chef_config.client_key, + client_name: chef_config.node_name) + end + end + end +end + diff --git a/lib/chef-dk/service_exceptions.rb b/lib/chef-dk/service_exceptions.rb index db4004745..e343747c2 100644 --- a/lib/chef-dk/service_exceptions.rb +++ b/lib/chef-dk/service_exceptions.rb @@ -121,6 +121,9 @@ class PolicyfileCleanError < PolicyfileNestedException class DeletePolicyGroupError < PolicyfileNestedException end + class PolicyCookbookCleanError < PolicyfileNestedException + end + class ChefRunnerError < StandardError include NestedExceptionWithInspector diff --git a/spec/unit/policyfile_services/clean_policy_cookbooks_spec.rb b/spec/unit/policyfile_services/clean_policy_cookbooks_spec.rb new file mode 100644 index 000000000..564be5dfb --- /dev/null +++ b/spec/unit/policyfile_services/clean_policy_cookbooks_spec.rb @@ -0,0 +1,261 @@ +# +# Copyright:: Copyright (c) 2015 Chef Software Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'spec_helper' +require 'chef-dk/policyfile_services/clean_policy_cookbooks' + +describe ChefDK::PolicyfileServices::CleanPolicyCookbooks do + + let(:cookbook_artifacts_list) do + { + "mysql" => { + "versions" => [ + { + "identifier" => "6b506252cae939c874bd59b560c339b01c31326b" + } + ] + }, + "build-essential" => { + "versions" => [ + { + "identifier" => "2db3df121028894f45497f847de91b91fbf76824" + }, + { + "identifier" => "d8ce58401d154378599b0fead81d2c390615602b" + } + ] + } + } + end + + let(:cookbook_ids_by_name) do + { + "mysql" => [ "6b506252cae939c874bd59b560c339b01c31326b" ], + "build-essential" => [ "2db3df121028894f45497f847de91b91fbf76824", "d8ce58401d154378599b0fead81d2c390615602b" ] + } + end + + let(:cookbook_ids_in_sets_by_name) do + cookbook_ids_by_name.inject({}) do |map, (name, id_list)| + map[name] = Set.new(id_list) + map + end + end + + let(:policies_list) do + { + "aar" => { + "revisions" => { + "37f9b658cdd1d9319bac8920581723efcc2014304b5f3827ee0779e10ffbdcc9" => { } + } + }, + "jenkins" => { + "revisions" => { + "613f803bdd035d574df7fa6da525b38df45a74ca82b38b79655efed8a189e073" => { }, + "6fe753184c8946052d3231bb4212116df28d89a3a5f7ae52832ad408419dd5eb" => { } + } + } + } + end + + let(:http_client) { instance_double(ChefDK::AuthenticatedHTTP) } + + let(:ui) { TestHelpers::TestUI.new } + + let(:chef_config) do + double("Chef::Config", + chef_server_url: "https://localhost:10443", + client_key: "/path/to/client/key.pem", + node_name: "deuce") + end + + subject(:clean_policy_cookbooks_service) do + described_class.new(ui: ui, config: chef_config) + end + + + it "configures an HTTP client with the user's credentials" do + expect(ChefDK::AuthenticatedHTTP).to receive(:new).with("https://localhost:10443", + signing_key_filename: "/path/to/client/key.pem", + client_name: "deuce") + clean_policy_cookbooks_service.http_client + end + + context "when an error occurs fetching cookbook data from the server" do + + let(:response) do + Net::HTTPResponse.send(:response_class, "500").new("1.0", "500", "Internal Server Error").tap do |r| + r.instance_variable_set(:@body, "oops") + end + end + + let(:http_exception) do + begin + response.error! + rescue => e + e + end + end + + before do + allow(clean_policy_cookbooks_service).to receive(:http_client).and_return(http_client) + expect(http_client).to receive(:get).with("/policies").and_return({}) + expect(http_client).to receive(:get).with("/cookbook_artifacts").and_raise(http_exception) + end + + it "raises a standardized nested exception" do + expect { clean_policy_cookbooks_service.run }.to raise_error(ChefDK::PolicyCookbookCleanError) + end + + end + + context "when the server returns cookbook data successfully" do + + before do + allow(clean_policy_cookbooks_service).to receive(:http_client).and_return(http_client) + + allow(http_client).to receive(:get).with("/cookbook_artifacts").and_return(cookbook_artifacts_list) + allow(http_client).to receive(:get).with("/policies").and_return(policies_list) + end + + context "when the server has no policy cookbooks" do + + let(:cookbook_artifacts_list) { {} } + let(:policies_list) { {} } + + it "has an empty list for all cookbooks" do + expect(clean_policy_cookbooks_service.all_cookbooks).to eq({}) + end + + it "has no in-use cookbook artifacts" do + expect(clean_policy_cookbooks_service.active_cookbooks).to eq({}) + end + + it "has no cookbooks to clean" do + expect(clean_policy_cookbooks_service.cookbooks_to_clean).to eq({}) + end + + it "does not clean any cookbooks" do + expect(http_client).to_not receive(:delete) + clean_policy_cookbooks_service.run + end + end + + context "when the server has policy cookbooks" do + + let(:policy_aar_37f9b65) do + { + "cookbook_locks" => { + "mysql" => { "identifier" => "6b506252cae939c874bd59b560c339b01c31326b" } + } + } + end + + let(:policy_jenkins_613f803) do + { + "cookbook_locks" => { + "mysql" => { "identifier" => "6b506252cae939c874bd59b560c339b01c31326b" }, + "build-essential" => { "identifier" => "2db3df121028894f45497f847de91b91fbf76824" } + } + } + end + + let(:policy_jenkins_6fe7531) do + { + "cookbook_locks" => { + "mysql" => { "identifier" => "6b506252cae939c874bd59b560c339b01c31326b" }, + "build-essential" => { "identifier" => "d8ce58401d154378599b0fead81d2c390615602b" } + } + } + end + + before do + allow(http_client).to receive(:get). + with("/policies/aar/revisions/37f9b658cdd1d9319bac8920581723efcc2014304b5f3827ee0779e10ffbdcc9"). + and_return(policy_aar_37f9b65) + allow(http_client).to receive(:get). + with("/policies/jenkins/revisions/613f803bdd035d574df7fa6da525b38df45a74ca82b38b79655efed8a189e073"). + and_return(policy_jenkins_613f803) + allow(http_client).to receive(:get). + with("/policies/jenkins/revisions/6fe753184c8946052d3231bb4212116df28d89a3a5f7ae52832ad408419dd5eb"). + and_return(policy_jenkins_6fe7531) + end + + + context "and all cookbooks are active" do + + it "lists all the cookbooks" do + expect(clean_policy_cookbooks_service.all_cookbooks).to eq(cookbook_ids_by_name) + end + + it "lists all active cookbooks" do + expect(clean_policy_cookbooks_service.active_cookbooks).to eq(cookbook_ids_in_sets_by_name) + end + + it "has no cookbooks to clean" do + expect(clean_policy_cookbooks_service.cookbooks_to_clean).to eq({}) + end + + it "does not clean any cookbooks" do + expect(http_client).to_not receive(:delete) + clean_policy_cookbooks_service.run + end + end + + context "and some cookbooks can be GC'd" do + + let(:policy_jenkins_6fe7531) do + { + "cookbook_locks" => { + "mysql" => { "identifier" => "6b506252cae939c874bd59b560c339b01c31326b" }, + # this is changed to reference the same cookbook as policy_jenkins_613f803 + "build-essential" => { "identifier" => "2db3df121028894f45497f847de91b91fbf76824" } + } + } + end + + let(:expected_active_cookbooks) do + { + "mysql" => Set.new([ "6b506252cae939c874bd59b560c339b01c31326b" ]), + "build-essential" => Set.new([ "2db3df121028894f45497f847de91b91fbf76824" ]) + } + end + + it "lists all the cookbooks" do + expect(clean_policy_cookbooks_service.all_cookbooks).to eq(cookbook_ids_by_name) + end + + it "lists all active cookbooks" do + expect(clean_policy_cookbooks_service.active_cookbooks).to eq(expected_active_cookbooks) + end + + it "lists non-active cookbooks" do + expected = { "build-essential" => Set.new([ "d8ce58401d154378599b0fead81d2c390615602b" ]) } + expect(clean_policy_cookbooks_service.cookbooks_to_clean).to eq(expected) + end + + it "deletes the non-active cookbooks" do + expect(http_client).to receive(:delete).with("/cookbook_artifacts/build-essential/d8ce58401d154378599b0fead81d2c390615602b") + clean_policy_cookbooks_service.run + end + + end + end + end + +end +