From ca43b2fceac33a8e39ab42f8e39d3a91965ed049 Mon Sep 17 00:00:00 2001 From: John Keiser Date: Mon, 28 Mar 2016 11:29:05 -0700 Subject: [PATCH] Build all gems using one Gemfile.lock --- .travis.yml | 7 +- Rakefile | 22 +-- appveyor.yml | 1 + omnibus/config/chefdk_gem_overrides.rb | 23 ---- omnibus/config/projects/chefdk.rb | 15 +-- omnibus/config/software/chef-dk-gemset.rb | 157 ++++++++++++++++++++++ omnibus/config/software/chefdk.rb | 89 ------------ tasks/dependencies.rb | 59 +------- 8 files changed, 182 insertions(+), 191 deletions(-) delete mode 100644 omnibus/config/chefdk_gem_overrides.rb create mode 100644 omnibus/config/software/chef-dk-gemset.rb delete mode 100644 omnibus/config/software/chefdk.rb diff --git a/.travis.yml b/.travis.yml index afa367453..7f1051000 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,8 +21,11 @@ before_install: - gem install bundler --no-ri --no-rdoc - gem --version -rvm: - - 2.1.1 +include: + - rvm: 2.2 + - rvm: 2.1 + - gemfile: chef-dk-gemset/Gemfile + rvm: 2.2 branches: only: diff --git a/Rakefile b/Rakefile index 8635823f9..a5060543e 100644 --- a/Rakefile +++ b/Rakefile @@ -17,8 +17,6 @@ require "bundler/gem_tasks" -require "github_changelog_generator/task" - namespace :version do task :bump => 'version:bump_patch' @@ -46,11 +44,17 @@ end require_relative 'tasks/dependencies' -GitHubChangelogGenerator::RakeTask.new :changelog do |config| - config.user = "chef" - config.project = "chef-dk" - config.future_release = ChefDK::VERSION - config.enhancement_labels = "enhancement,Enhancement,New Feature,Feature".split(",") - config.bug_labels = "bug,Bug,Improvement,Upstream Bug".split(",") - config.exclude_labels = "duplicate,question,invalid,wontfix,no_changelog,Exclude From Changelog,Question,Discussion".split(",") +begin + require "github_changelog_generator/task" + + GitHubChangelogGenerator::RakeTask.new :changelog do |config| + config.user = "chef" + config.project = "chef-dk" + config.future_release = ChefDK::VERSION + config.enhancement_labels = "enhancement,Enhancement,New Feature,Feature".split(",") + config.bug_labels = "bug,Bug,Improvement,Upstream Bug".split(",") + config.exclude_labels = "duplicate,question,invalid,wontfix,no_changelog,Exclude From Changelog,Question,Discussion".split(",") + end +rescue LoadError + puts "github_changelog_generator is not available. gem install github_changelog_generator to generate changelogs" end diff --git a/appveyor.yml b/appveyor.yml index 5796662a3..6cff67e32 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -23,6 +23,7 @@ install: - gem --version - gem install bundler --no-ri --no-rdoc - bundler --version + - set BUNDLE_GEMFILE=C:\projects\chefdk\chef-dk-gemset\Gemfile build_script: - bundle install diff --git a/omnibus/config/chefdk_gem_overrides.rb b/omnibus/config/chefdk_gem_overrides.rb deleted file mode 100644 index 2f9852847..000000000 --- a/omnibus/config/chefdk_gem_overrides.rb +++ /dev/null @@ -1,23 +0,0 @@ -override :berkshelf, :version => "v4.3.1" -override :bundler, :version => "1.11.2" -override :chef, :version => "12.8.1" -override :"chef-provisioning", :version => "v1.6.0" -override :"chef-provisioning-aws", :version => "v1.8.0" -override :"chef-provisioning-azure", :version => "v0.5.0" -override :"chef-provisioning-fog", :version => "v0.16.0" -override :"chef-provisioning-vagrant", :version => "v0.11.0" -override :"chef-vault", :version => "v2.8.0" -override :chefspec, :version => "v4.6.0" -override :"dep-selector-libgecode", :version => "1.2.0" -override :fauxhai, :version => "v3.1.0" -override :foodcritic, :version => "v6.0.1" -override :inspec, :version => "v0.16.3" -override :"kitchen-inspec", :version => "v0.12.5" -override :"kitchen-vagrant", :version => "v0.19.0" -override :"knife-windows", :version => "v1.4.0" -override :"knife-spork", :version => "1.6.1" -override :nokogiri, :version => "1.6.7.2" -override :ohai, :version => "v8.13.0" -override :pry, :version => "0.10.3" -override :"test-kitchen", :version => "v1.6.0" -override :"winrm-fs", :version => "v0.3.2" diff --git a/omnibus/config/projects/chefdk.rb b/omnibus/config/projects/chefdk.rb index 29d05ff5f..af46e8fd8 100644 --- a/omnibus/config/projects/chefdk.rb +++ b/omnibus/config/projects/chefdk.rb @@ -32,10 +32,6 @@ install_dir "#{default_root}/#{name}" end -# Bring in generated gem overrides -overrides_file = File.expand_path('../../chefdk_gem_overrides.rb', __FILE__) -instance_eval(IO.read(overrides_file), overrides_file, 1) - # Lower level library pins override :libedit, version: "20130712-3.1" #override :"libffi", version: "3.2.1" @@ -49,9 +45,10 @@ override :makedepend, version: "1.0.5" #override :"ncurses", version: "5.9" #override :"pkg-config-lite", version: "0.28-1" -override :rubocop, version: "v0.37.2" override :ruby, version: "2.1.8" override :rubygems, version: "2.5.2" +override :bundler, version: "1.11.2" +override :appbundler, version: "jk/multiple-gems" override :"util-macros", version: "1.19.0" override :xproto, version: "7.0.28" #override :"libyaml", version: "0.1.6" @@ -64,13 +61,7 @@ #override :"openssl" # leave this? dependency "preparation" -dependency "rb-readline" -dependency "chefdk" -dependency "pry" -dependency "chef-provisioning-aws" -dependency "chef-provisioning-fog" -dependency "chef-provisioning-vagrant" -dependency "chef-provisioning-azure" +dependency "chef-dk-gemset" dependency "ruby-windows-devkit" if windows? dependency "rubygems-customization" dependency "shebang-cleanup" diff --git a/omnibus/config/software/chef-dk-gemset.rb b/omnibus/config/software/chef-dk-gemset.rb new file mode 100644 index 000000000..d426f211b --- /dev/null +++ b/omnibus/config/software/chef-dk-gemset.rb @@ -0,0 +1,157 @@ +# +# Copyright 2012-2014 Chef Software, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name "chef-dk-gemset" +default_version "local_source" + +# For the specific super-special version "local_source", build the source from +# the local git checkout. This is what you'd want to occur by default if you +# just ran omnibus build locally. +version("local_source") do + source path: File.expand_path("../..", project.files_path), + # Since we are using the local repo, we try to not copy any files + # that are generated in the process of bundle installing omnibus. + # If the install steps are well-behaved, this should not matter + # since we only perform bundle and gem installs from the + # omnibus cache source directory, but we do this regardless + # to maintain consistency between what a local build sees and + # what a github based build will see. + options: { exclude: [ "omnibus/vendor" ] } +end + +# For any version other than "local_source", fetch from github. +if version != "local_source" + source git: "git://github.com/chef/chef-dk.git" +end + +relative_path "chef-dk" + +dependency "ruby" +dependency "rubygems" +dependency "bundler" + +# For nokogiri +dependency "libxml2" +dependency "libxslt" +dependency "libiconv" +dependency "liblzma" +dependency "zlib" + +# For berkshelf +dependency "libarchive" + +dependency "chefdk-env-customization" if windows? + +dependency "appbundler" + +require "shellwords" + +build do + def self.embedded_bin(bin) + windows_safe_path("#{install_dir}/embedded/bin/#{bin}") + end + + chef_dk_gemset = File.join(project_dir, "chef-dk-gemset") + + env = with_standard_compiler_flags(with_embedded_path, bfd_flags: true).merge( + "BUNDLE_GEMFILE" => File.join(chef_dk_gemset, "Gemfile") + ) + + # From dep-selector-libgecode + # On some RHEL-based systems, the default GCC that's installed is 4.1. We + # need to use 4.4, which is provided by the gcc44 and gcc44-c++ packages. + # These do not use the gcc binaries so we set the flags to point to the + # correct version here. + if File.exist?("/usr/bin/gcc44") + env["CC"] = "gcc44" + env["CXX"] = "g++44" + end + + # From dep-selector-libgecode + # Ruby DevKit ships with BSD Tar + env["PROG_TAR"] = "bsdtar" if windows? + env["ARFLAGS"] = "rv #{env["ARFLAGS"]}" if env["ARFLAGS"] + + # Set up nokogiri environment and args + env["NOKOGIRI_USE_SYSTEM_LIBRARIES"] = "true" + block do + nokogiri_build_config = [ + "--use-system-libraries", + "--with-xml2-lib=#{install_dir}/embedded/lib", + "--with-xml2-include=#{install_dir}/embedded/include/libxml2", + "--with-xslt-lib=#{install_dir}/embedded/lib", + "--with-xslt-include=#{install_dir}/embedded/include/libxslt", + "--with-iconv-dir=#{install_dir}/embedded", + "--with-zlib-dir=#{install_dir}/embedded" + ].join(" ") + + FileUtils.mkdir_p("#{chef_dk_gemset}/.bundle") + # bundle config build.nokogiri #{nokogiri_build_config} messes up the line + IO.write("#{chef_dk_gemset}/.bundle/config", <<-EOM.gsub(/^\s+/, '')) + --- + BUNDLE_BUILD__NOKOGIRI: #{nokogiri_build_config} + EOM + end + + # First, bundle install to bring over all dependencies + bundle "install --without=development --jobs=4 --retry=4 --verbose --frozen", env: env + + # Next, build and install the chef-dk and chef-dk-gemset gems + bundle "exec rake build", env: env + bundle "exec rake build", cwd: chef_dk_gemset, env: env + gem "install pkg/chef-dk-*.gem chef-dk-gemset/pkg/chef-dk-*.gem --no-document --ignore-dependencies", env: env + + # Ensure the main bin dir exists + bin_dir = File.join(install_dir, "bin") + FileUtils.mkdir_p(bin_dir) + + # appbundle all the things + appbundler_bin = embedded_bin('appbundler') + appbundler_args = [ chef_dk_gemset, bin_dir ] + appbundler_args += %w(berkshelf chef chef-dk chef-vault foodcritic ohai rubocop test-kitchen) + appbundler_args = appbundler_args.map { |a| ::Shellwords.escape(a) } + command "#{appbundler_bin} #{appbundler_args.join(" ")}", env: env + + # + # Everything that follows is from the chef software def: + # + + # + # TODO bundle show chef can get the installed location of chef. Install from there. + # TODO TODO just copy the stuff from the chef install over. + if windows? + block do + # Copy the chef gem's distro stuff over + chef_gem_path = shellout!("#{embedded_bin("bundle")} show chef", env: env).stdout.chomp + mkdir "#{install_dir}/modules/chef" + copy "#{chef_gem_path}/distro/powershell/chef/*", "#{install_dir}/modules/chef" + end + end + + # TODO find a way to get this in the gemspec + unless aix? || windows? + gem "install ruby-shadow --no-document --verbose", env: env + end + + # Clean up + delete "#{install_dir}/embedded/docs" + delete "#{install_dir}/embedded/share/man" + delete "#{install_dir}/embedded/share/doc" + delete "#{install_dir}/embedded/share/gtk-doc" + delete "#{install_dir}/embedded/ssl/man" + delete "#{install_dir}/embedded/man" + delete "#{install_dir}/embedded/info" +end diff --git a/omnibus/config/software/chefdk.rb b/omnibus/config/software/chefdk.rb deleted file mode 100644 index 1265a7036..000000000 --- a/omnibus/config/software/chefdk.rb +++ /dev/null @@ -1,89 +0,0 @@ -# -# Copyright 2012-2014 Chef Software, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -name "chefdk" -default_version "local_source" - -# For the specific super-special version "local_source", build the source from -# the local git checkout. This is what you'd want to occur by default if you -# just ran omnibus build locally. -version("local_source") do - source path: "#{project.files_path}/../..", - # Since we are using the local repo, we try to not copy any files - # that are generated in the process of bundle installing omnibus. - # If the install steps are well-behaved, this should not matter - # since we only perform bundle and gem installs from the - # omnibus cache source directory, but we do this regardless - # to maintain consistency between what a local build sees and - # what a github based build will see. - options: { exclude: [ "omnibus/vendor" ] } -end - -# For any version other than "local_source", fetch from github. -if version != "local_source" - source git: "git://github.com/chef/chef-dk.git" -end - -relative_path "chef-dk" - -dependency "ruby" - -dependency "rubygems" -dependency "bundler" -dependency "appbundler" -dependency "chef" -dependency "test-kitchen" -dependency "inspec" -dependency "kitchen-inspec" -dependency "kitchen-vagrant" -dependency "berkshelf" -dependency "chef-vault" -dependency "foodcritic" -dependency "ohai" -dependency "rubocop" -# This is a TK dependency but isn't declared in that software definition -# because it is an optional dependency but we want to give it to ChefDK users -dependency "winrm-transport" -dependency "openssl-customization" -dependency "knife-windows" -dependency "knife-spork" -dependency "fauxhai" -dependency "chefspec" -dependency "chef-provisioning" - -dependency "chefdk-env-customization" if windows? - -build do - env = with_standard_compiler_flags(with_embedded_path).merge( - # Rubocop pulls in nokogiri 1.5.11, so needs PKG_CONFIG_PATH and - # NOKOGIRI_USE_SYSTEM_LIBRARIES until rubocop stops doing that - "PKG_CONFIG_PATH" => "#{install_dir}/embedded/lib/pkgconfig", - "NOKOGIRI_USE_SYSTEM_LIBRARIES" => "true", - ) - - bundle "install", env: env - gem "build chef-dk.gemspec", env: env - gem "install chef-dk*.gem" \ - " --no-ri --no-rdoc" \ - " --verbose", env: env - - appbundle 'berkshelf' - appbundle 'chefdk' - appbundle 'chef-vault' - appbundle 'foodcritic' - appbundle 'rubocop' - appbundle 'test-kitchen' -end diff --git a/tasks/dependencies.rb b/tasks/dependencies.rb index c6741940f..954347b07 100644 --- a/tasks/dependencies.rb +++ b/tasks/dependencies.rb @@ -1,60 +1,7 @@ -class OverrideReader - def overrides - @overrides ||= {} - end - def override(name, **options) - overrides[name] = options - end -end - task :dependencies do - # Read the chefdk overrides file to get the list of gems and other things besides version (like source info) - overrides_file = File.expand_path("../../omnibus/config/chefdk_gem_overrides.rb", __FILE__) - puts "Reading #{overrides_file} ..." - reader = OverrideReader.new - reader.instance_eval(IO.read(overrides_file), overrides_file, 1) - - # Grab all matching specs - found_gems = {} - sources = Gem::SourceList.from [ "https://rubygems.org" ] - sources.each_source { |s| s.load_specs(:latest) } - specs = Gem::SpecFetcher.new(sources).detect(:latest) do |tuple| - name = tuple.name.to_sym - name = :'rubygems' if name == :'rubygems-update' - reader.overrides.has_key?(name) && tuple.platform === 'ruby' - end - specs.each do |tuple, source| - name = tuple.name.to_sym - name = :'rubygems' if name == :'rubygems-update' - # :latest will give you different versions depending on arch. Pick latest latest. - if !found_gems[name] || tuple.version >= found_gems[name] - found_gems[name] = tuple.version - end - end - - # Figure out the final override version for each override - new_overrides = {} - reader.overrides.each do |name, version: nil, **options| - new_version = found_gems[name].to_s - raise "Could not find #{gem_name} in rubygems!" unless new_version - new_version = "v#{new_version}" if version.start_with?("v") - if version == new_version - puts "#{name}: #{new_version}" - else - puts "#{name}: #{new_version} (was #{version})" - end - new_overrides[name] = { version: new_version, **options } - end - - # Write the file back out - puts "Writing changes out to #{overrides_file} ..." - output = File.open(overrides_file, 'w') - new_overrides.each do |name, **options| - line = "override #{name.inspect}" - options.each do |key, value| - line << ", #{key.inspect} => #{value.inspect}" + Dir.chdir(File.expand_path("../../chef-dk-gemset", __FILE__)) do + Bundler.with_clean_env do + sh "bundle update" end - output.puts line end - output.close end