Export archive

chef-dk.gemspec

@@ -41,6 +41,8 @@ Gem::Specification.new do |gem|
   gem.add_dependency "mixlib-shellout", ">= 2.0.0.rc.0", "< 3.0.0"
   gem.add_dependency "ffi-yajl", ">= 1.0", "< 3.0"
 
+  gem.add_dependency "minitar", "~> 0.5.4"
+
   gem.add_dependency "chef", "~> 12.0", ">= 12.2.1"
 
   gem.add_dependency "solve", "~> 2.0", ">= 2.0.1"

lib/chef-dk/command/export.rb

@@ -48,6 +48,13 @@ class Export < Base
 
 E
 
+      option :archive,
+        short:       "-a",
+        long:        "--archive",
+        description: "Export as a tarball archive rather than a directory",
+        default:     false,
+        boolean:     true
+
       option :force,
         short:       "-f",
         long:        "--force",
@@ -79,7 +86,7 @@ def initialize(*args)
       def run(params = [])
         return 1 unless apply_params!(params)
         export_service.run
-        ui.msg("Exported policy '#{export_service.policyfile_lock.name}' to #{export_dir}")
+        ui.msg("Exported policy '#{export_service.policyfile_lock.name}' to #{export_target}")
         0
       rescue ExportDirNotEmpty => e
         ui.err("ERROR: " + e.message)
@@ -90,14 +97,27 @@ def run(params = [])
         1
       end
 
+      def export_target
+        if archive?
+          export_service.archive_file_location
+        else
+          export_dir
+        end
+      end
+
       def debug?
         !!config[:debug]
       end
 
+      def archive?
+        !!config[:archive]
+      end
+
       def export_service
         @export_service ||= PolicyfileServices::ExportRepo.new(policyfile: policyfile_relative_path,
                                                        export_dir: export_dir,
                                                        root_dir: Dir.pwd,
+                                                       archive: archive?,
                                                        force: config[:force])
       end
 

lib/chef-dk/policyfile_services/export_repo.rb

@@ -16,6 +16,10 @@
 #
 
 require 'fileutils'
+require 'tmpdir'
+require 'zlib'
+
+require 'archive/tar/minitar'
 
 require 'chef-dk/service_exceptions'
 require 'chef-dk/policyfile_lock'
@@ -38,9 +42,10 @@ class ExportRepo
       attr_reader :root_dir
       attr_reader :export_dir
 
-      def initialize(policyfile: nil, export_dir: nil, root_dir: nil, force: false)
+      def initialize(policyfile: nil, export_dir: nil, root_dir: nil, archive: false, force: false)
         @root_dir = root_dir
         @export_dir = File.expand_path(export_dir)
+        @archive = archive
         @force_export = force
 
         @policy_data = nil
@@ -49,6 +54,12 @@ def initialize(policyfile: nil, export_dir: nil, root_dir: nil, force: false)
         policyfile_rel_path = policyfile || "Policyfile.rb"
         policyfile_full_path = File.expand_path(policyfile_rel_path, root_dir)
         @storage_config = Policyfile::StorageConfig.new.use_policyfile(policyfile_full_path)
+
+        @staging_dir = nil
+      end
+
+      def archive?
+        @archive
       end
 
       def policy_name
@@ -74,24 +85,59 @@ def policyfile_lock
         @policyfile_lock || validate_lockfile
       end
 
+      def archive_file_location
+        return nil unless archive?
+        filename = "#{policyfile_lock.name}-#{policyfile_lock.revision_id}.tgz"
+        File.join(export_dir, filename)
+      end
+
       def export
-        create_repo_structure
-        copy_cookbooks
-        create_policyfile_data_item
+        with_staging_dir do
+          create_repo_structure
+          copy_cookbooks
+          create_policyfile_data_item
+          if archive?
+            create_archive
+          else
+            mv_staged_repo
+          end
+        end
       rescue => error
         msg = "Failed to export policy (in #{policyfile_filename}) to #{export_dir}"
         raise PolicyfileExportRepoError.new(msg, error)
       end
 
       private
 
-      def create_repo_structure
-        FileUtils.rm_rf(cookbooks_dir)
-        FileUtils.rm_rf(policyfiles_data_bag_dir)
+      def with_staging_dir
+        p = Process.pid
+        t = Time.new.utc.strftime("%Y%m%d%H%M%S")
+        Dir.mktmpdir("chefdk-export-#{p}-#{t}") do |d|
+          begin
+            @staging_dir = d
+            yield
+          ensure
+            @staging_dir = nil
+          end
+        end
+      end
+
+      def create_archive
+        Zlib::GzipWriter.open(archive_file_location) do |gz_file|
+          Dir.chdir(staging_dir) do
+            Archive::Tar::Minitar.pack(".", gz_file)
+          end
+        end
+      end
+
+      def staging_dir
+        @staging_dir
+      end
 
+      def create_repo_structure
         FileUtils.mkdir_p(export_dir)
-        FileUtils.mkdir_p(cookbooks_dir)
-        FileUtils.mkdir_p(policyfiles_data_bag_dir)
+        FileUtils.mkdir_p(cookbooks_staging_dir)
+        FileUtils.mkdir_p(policyfiles_data_bag_staging_dir)
       end
 
       def copy_cookbooks
@@ -102,7 +148,7 @@ def copy_cookbooks
 
       def copy_cookbook(lock)
         dirname = "#{lock.name}-#{lock.dotted_decimal_identifier}"
-        export_path = File.join(export_dir, "cookbooks", dirname)
+        export_path = File.join(staging_dir, "cookbooks", dirname)
         metadata_rb_path = File.join(export_path, "metadata.rb")
         FileUtils.cp_r(lock.cookbook_path, export_path)
         FileUtils.rm_f(metadata_rb_path)
@@ -117,8 +163,6 @@ def copy_cookbook(lock)
       end
 
       def create_policyfile_data_item
-        policy_id = "#{policyfile_lock.name}-#{POLICY_GROUP}"
-        item_path = File.join(export_dir, "data_bags", "policyfiles", "#{policy_id}.json")
 
         lock_data = policyfile_lock.to_lock.dup
 
@@ -139,6 +183,17 @@ def create_policyfile_data_item
         end
       end
 
+      def mv_staged_repo
+        # If we got here, either these dirs are empty/don't exist or force is
+        # set to true.
+        FileUtils.rm_rf(cookbooks_dir)
+        FileUtils.rm_rf(policyfiles_data_bag_dir)
+
+        FileUtils.mv(cookbooks_staging_dir, export_dir)
+        FileUtils.mkdir_p(export_data_bag_dir)
+        FileUtils.mv(policyfiles_data_bag_staging_dir, export_data_bag_dir)
+      end
+
       def validate_lockfile
         return @policyfile_lock if @policyfile_lock
         @policyfile_lock = ChefDK::PolicyfileLock.new(storage_config).build_from_lock_data(policy_data)
@@ -164,7 +219,7 @@ def assert_lockfile_exists!
       end
 
       def assert_export_dir_clean!
-        if !force_export? && !conflicting_fs_entries.empty?
+        if !force_export? && !conflicting_fs_entries.empty? && !archive?
           msg = "Export dir (#{export_dir}) not clean. Refusing to export. (Conflicting files: #{conflicting_fs_entries.join(', ')})"
           raise ExportDirNotEmpty, msg
         end
@@ -183,8 +238,28 @@ def cookbooks_dir
         File.join(export_dir, "cookbooks")
       end
 
+      def export_data_bag_dir
+        File.join(export_dir, "data_bags")
+      end
+
       def policyfiles_data_bag_dir
-        File.join(export_dir, "data_bags", "policyfiles")
+        File.join(export_data_bag_dir, "policyfiles")
+      end
+
+      def policy_id
+        "#{policyfile_lock.name}-#{POLICY_GROUP}"
+      end
+
+      def item_path
+        File.join(staging_dir, "data_bags", "policyfiles", "#{policy_id}.json")
+      end
+
+      def cookbooks_staging_dir
+        File.join(staging_dir, "cookbooks")
+      end
+
+      def policyfiles_data_bag_staging_dir
+        File.join(staging_dir, "data_bags", "policyfiles")
       end
 
     end

spec/unit/command/export_spec.rb

@@ -59,6 +59,19 @@
       end
     end
 
+    context "when archive mode is set" do
+
+      let(:params) { [ "path/to/export", "-a" ] }
+
+      it "enables archiving the exported repo" do
+        expect(command.archive?).to be(true)
+      end
+
+      it "configures the export service to archive" do
+        expect(command.export_service.archive?).to be(true)
+      end
+    end
+
     context "when the path to the exported repo is given" do
 
       let(:params) { [ "path/to/export" ] }

spec/unit/policyfile_services/export_repo_spec.rb

@@ -40,10 +40,13 @@
 
   let(:force_export) { false }
 
+  let(:archive) { false }
+
   subject(:export_service) do
     described_class.new(policyfile: policyfile_rb_explicit_name,
                         root_dir: working_dir,
                         export_dir: export_dir,
+                        archive: archive,
                         force: force_export)
   end
 
@@ -154,6 +157,22 @@
         expect(export_service.policy_name).to eq("install-example")
       end
 
+      context "when using archive mode" do
+
+        let(:archive) { true }
+
+        # TODO: also support a full file name
+        context "when the given 'export_dir' is a directory" do
+
+          it "sets the archive file location to $policy_name-$revision.tgz" do
+            expected = File.join(export_dir, "install-example-60e5ad638dce219d8f87d589463ec4a9884007ba5e2adbb4c0a7021d67204f1a.tgz")
+            expect(export_service.archive_file_location).to eq(expected)
+          end
+
+        end
+
+      end
+
       describe "writing updates to the policyfile lock" do
 
         let(:updated_lockfile_io) { StringIO.new }
@@ -169,7 +188,7 @@
 
       context "copying the cookbooks to the export dir" do
 
-        context "when the export dir is empty" do
+        shared_examples_for "successful_export" do
           before do
             allow(export_service.policyfile_lock).to receive(:validate_cookbooks!).and_return(true)
             export_service.run
@@ -222,6 +241,11 @@
 
         end
 
+        context "when the export dir is empty" do
+
+          include_examples "successful_export"
+        end
+
         context "When an error occurs creating the export" do
 
           before do
@@ -253,14 +277,15 @@
           end
 
           it "ignores the non-conflicting content and exports" do
-            export_service.run
-
             expect(File).to exist(file_in_export_dir)
             expect(File).to exist(extra_data_bag_item)
 
             expect(File).to be_directory(File.join(export_dir, "cookbooks"))
             expect(File).to be_directory(File.join(export_dir, "data_bags"))
           end
+
+          include_examples "successful_export"
+
         end
 
         context "When the export dir has conflicting content" do
@@ -310,9 +335,62 @@
 
           end
 
-        end
+        end # When the export dir has conflicting content
 
-      end
+        context "when archive mode is enabled" do
+
+          let(:archive) { true }
+
+          let(:expected_archive_path) do
+            File.join(export_dir, "install-example-60e5ad638dce219d8f87d589463ec4a9884007ba5e2adbb4c0a7021d67204f1a.tgz")
+          end
+
+          it "exports the repo as a tgz archive" do
+            expect(File).to exist(expected_archive_path)
+          end
+
+          include_examples "successful_export" do
+
+            # explode the tarball so the assertions can find the files
+            before do
+              Zlib::GzipReader.open(expected_archive_path) do |gz_file|
+                tar = Archive::Tar::Minitar::Input.new(gz_file)
+                tar.each do |e|
+                  tar.extract_entry(export_dir, e)
+                end
+              end
+            end
+
+          end
+
+          context "when the target dir has a cookbooks or data_bags dir" do
+
+            let(:cookbooks_dir) { File.join(export_dir, "cookbooks") }
+
+            let(:file_in_cookbooks_dir) { File.join(cookbooks_dir, "some_random_cruft") }
+
+            let(:policyfiles_data_bag_dir) { File.join(export_dir, "data_bags", "policyfiles") }
+
+            let(:extra_policyfile_data_item) { File.join(policyfiles_data_bag_dir, "leftover-policy.json") }
+
+            before do
+              FileUtils.mkdir_p(export_dir)
+              FileUtils.mkdir_p(cookbooks_dir)
+              FileUtils.mkdir_p(policyfiles_data_bag_dir)
+              File.open(file_in_cookbooks_dir, "wb+") { |f| f.print "some random cruft" }
+              File.open(extra_policyfile_data_item, "wb+") { |f| f.print "some random cruft" }
+            end
+
+            it "exports successfully" do
+              expect { export_service.run }.to_not raise_error
+              expect(File).to exist(expected_archive_path)
+            end
+
+          end
+
+        end # when archive mode is enabled
+
+      end # copying the cookbooks to the export dir
     end
 
   end