From ff19105eb8f264a1bdc89d4cee099ac5a320b78d Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Mon, 27 Apr 2020 08:26:42 -0500 Subject: [PATCH] Add Hatchet Regression Tests (#969) I went through all the tests in the Ruby buildpack and cataloged what I think can be generalized between languages to prevent regressions. This PR add tests for these cases: * Test CI deploys run tests and use the cache [[reference test](https://github.com/heroku/heroku-buildpack-ruby/blob/f488bd53c7ff0b78e17c2405166cbd4a3af75ee2/spec/hatchet/ci_spec.rb#L36)] * Test cache for regular deploys is used on repeated deploys (This was already tested on the Python buildpack, I moved it) [[reference test](https://github.com/heroku/heroku-buildpack-ruby/blob/e34c583c139911d059f5627bb25125707288f053/spec/hatchet/stack_spec.rb#L21-L25)] * Test modifying a requirement clears the cache appropriately (This was already tested on the Python buildpack, I moved it) * Test deploying the getting started guide works [[reference test](https://github.com/heroku/heroku-buildpack-ruby/blob/424a7245e2da86845a20d58a9482bcf2a00c3a8f/spec/hatchet/getting_started_spec.rb#L5)] * Test that all paths set by the buildpack are absolute instead of relative [[reference test](https://github.com/heroku/heroku-buildpack-ruby/blob/249d3c1a4e97068f8fd016f10fa0839709d95658/spec/hatchet/rails5_spec.rb#L68]) * Test upgrading stack invalidates the cache [[reference test](https://github.com/heroku/heroku-buildpack-ruby/blob/f488bd53c7ff0b78e17c2405166cbd4a3af75ee2/spec/hatchet/stack_spec.rb#L3)] * Test that builds fail when a bad version is specified [[reference test](https://github.com/heroku/heroku-buildpack-ruby/blob/249d3c1a4e97068f8fd016f10fa0839709d95658/spec/hatchet/ruby_spec.rb#L5)] In addition to that I've also got a CNB test with `pack-build` with the getting started app, but since python isn't `cnb` capable yet I didn't add one. --- .travis.yml | 7 ++- Gemfile | 2 + Gemfile.lock | 25 +++++--- hatchet.json | 3 +- hatchet.lock | 4 +- spec/hatchet/ci_spec.rb | 33 +++++++++++ spec/hatchet/python_spec.rb | 112 +++++++++++++++++++----------------- spec/spec_helper.rb | 7 +-- 8 files changed, 119 insertions(+), 74 deletions(-) create mode 100644 spec/hatchet/ci_spec.rb diff --git a/.travis.yml b/.travis.yml index 8c6a51a22..92b74aeb9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ branches: - master rvm: -- 2.4.4 +- 2.6.6 before_script: - gem install bundler -v 1.16.2 @@ -28,7 +28,8 @@ jobs: name: Run Hatchet script: - bundle exec hatchet ci:setup - - bundle exec rspec + - PARALLEL_SPLIT_TEST_PROCESSES=11 bundle exec parallel_split_test spec/hatchet/ + env: matrix: - TESTFOLDER=test/run-deps @@ -37,7 +38,7 @@ env: global: - HATCHET_RETRIES=3 - IS_RUNNING_ON_CI=true - - HATCHET_APP_LIMIT=5 + - HATCHET_APP_LIMIT=80 - HATCHET_DEPLOY_STRATEGY=git - secure: yjtlPE5FbVxTKnjUy/tZUBgSEf4qADD3QOxtgziuid73S0U/1IEXlMGFULsQzIjtlHKmHeywZqpVVEpthIH4RuT7uoX1Pb7SSM/g0T8fT3VoEFbFK1uYl0oZQbUS4Klxv9tPiumj8if3m6ULEGIz1X0wZcMOC0tMLwVCnwmap0E= - secure: ZeFTHWwnpIKE9nAqs88ocmiQh7bKce84lilGm5J23nf3N6V4wNyLwqlkvsM008WGBCaOg9AUx7ZunasT0ANsR5gLP3eV2UUg7ILdRgV2Gy13eNRFheC4PHdN92RqQ3aKoqlIv2K999xlhVjod0NzhkQQXB6PddfQINbuU7ks6As= diff --git a/Gemfile b/Gemfile index 9b336245f..0d1a9c26b 100644 --- a/Gemfile +++ b/Gemfile @@ -4,3 +4,5 @@ gem "rspec" gem "heroku_hatchet" gem "rspec-retry" gem "rake" +gem "parallel_split_test" + diff --git a/Gemfile.lock b/Gemfile.lock index 6a6f314b6..f10294056 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,21 +1,22 @@ GEM remote: https://rubygems.org/ specs: - activesupport (5.2.1) + activesupport (6.0.2.2) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 0.7, < 2) minitest (~> 5.1) tzinfo (~> 1.1) - concurrent-ruby (1.1.3) + zeitwerk (~> 2.2) + concurrent-ruby (1.1.6) diff-lcs (1.3) erubis (2.7.0) - excon (0.62.0) + excon (0.73.0) heroics (0.0.25) erubis (~> 2.0) excon moneta multi_json (>= 1.9.2) - heroku_hatchet (4.0.6) + heroku_hatchet (5.0.2) excon (~> 0) minitest-retry (~> 0.1.9) platform-api (~> 2) @@ -23,13 +24,17 @@ GEM rrrretry (~> 1) thor (~> 0) threaded (~> 0) - i18n (1.1.1) + i18n (1.8.2) concurrent-ruby (~> 1.0) - minitest (5.11.3) + minitest (5.14.0) minitest-retry (0.1.9) minitest (>= 5.0) moneta (1.0.0) - multi_json (1.13.1) + multi_json (1.14.1) + parallel (1.19.1) + parallel_split_test (0.7.0) + parallel (>= 0.5.13) + rspec (>= 3.1.0) platform-api (2.2.0) heroics (~> 0.0.25) moneta (~> 1.0.0) @@ -55,17 +60,19 @@ GEM thor (0.20.3) thread_safe (0.3.6) threaded (0.0.4) - tzinfo (1.2.5) + tzinfo (1.2.7) thread_safe (~> 0.1) + zeitwerk (2.3.0) PLATFORMS ruby DEPENDENCIES heroku_hatchet + parallel_split_test rake rspec rspec-retry BUNDLED WITH - 1.16.3 + 2.1.4 diff --git a/hatchet.json b/hatchet.json index 20ed7dabd..ea0970523 100644 --- a/hatchet.json +++ b/hatchet.json @@ -1,5 +1,6 @@ { "python": [ - "heroku/python-getting-started" + "heroku/python-getting-started", + "sharpstone/python_default" ] } diff --git a/hatchet.lock b/hatchet.lock index 1dbcd8de5..32e9dae10 100644 --- a/hatchet.lock +++ b/hatchet.lock @@ -1,3 +1,5 @@ --- - - "./repos/python/python-getting-started" - - 443a90c58be6881583cd7ef628e3869e3c30bb98 + - master +- - "./repos/python/python_default" + - ca947f69027b2a30be5d26f9a42f25e54f4d7a1a diff --git a/spec/hatchet/ci_spec.rb b/spec/hatchet/ci_spec.rb new file mode 100644 index 000000000..fe4bf3f4e --- /dev/null +++ b/spec/hatchet/ci_spec.rb @@ -0,0 +1,33 @@ +require_relative '../spec_helper' + +describe "Heroku CI" do + it "works" do + before_deploy = Proc.new do + File.open("app.json", "w+") do |f| + f.puts <<~EOM + { + "environments": { + "test": { + "scripts": { + "test": "nosetests" + } + } + } + } + EOM + end + + run!("echo nose >> requirements.txt") + end + + Hatchet::Runner.new("python_default", before_deploy: before_deploy).run_ci do |test_run| + expect(test_run.output).to match("Downloading nose") + expect(test_run.output).to match("OK") + + test_run.run_again + + expect(test_run.output).to match("installing from cache") + expect(test_run.output).to_not match("Downloading nose") + end + end +end diff --git a/spec/hatchet/python_spec.rb b/spec/hatchet/python_spec.rb index 66023bfac..83bb7a1f2 100644 --- a/spec/hatchet/python_spec.rb +++ b/spec/hatchet/python_spec.rb @@ -1,68 +1,72 @@ require_relative '../spec_helper' -describe "Default Python Deploy" do +describe "Python" do + describe "cache" do + it "functions correctly" do + Hatchet::Runner.new("python_default").deploy do |app| + expect(app.output).to match(/Installing pip/) - def set_python_version(d, v) - Dir.chdir(d) do - File.open('runtime.txt', 'w') do |f| - f.puts "python-#{v}" - end - `git add runtime.txt && git commit -am "setting python version"` - end - end - - before(:each) do - set_python_version(app.directory, python_version) - init_app(app) - end + expect(app.output).to_not match("Requirements file has been changed, clearing cached dependencies") + expect(app.output).to_not match("No change in requirements detected, installing from cache") + expect(app.output).to_not match("No such file or directory") + expect(app.output).to_not match("cp: cannot create regular file") - ["3.7.6", "3.8.2"].each do |version| - context "on python-#{version}" do - let(:app) { Hatchet::Runner.new('python-getting-started', stack: DEFAULT_STACK) } - let(:python_version) { version } - it "🐍" do - app.deploy do |app| - # What should happen on first deploy - expect(app.output).to match(/Installing pip/) + # Redeploy with changed requirements file + run!(%Q{echo "" >> requirements.txt}) + run!(%Q{echo "pygments" >> requirements.txt}) + run!(%Q{git add . ; git commit --allow-empty -m next}) + app.push! - # What should not happen - expect(app.output).to_not match("Requirements file has been changed, clearing cached dependencies") - expect(app.output).to_not match("No change in requirements detected, installing from cache") - expect(app.output).to_not match("No such file or directory") - expect(app.output).to_not match("cp: cannot create regular file") + # Check the cache to have cleared + expect(app.output).to match("Requirements file has been changed, clearing cached dependencies") + expect(app.output).to_not match("No dependencies found, preparing to install") + expect(app.output).to_not match("No change in requirements detected, installing from cache") - # let the previous build finish - sleep(5) + # With no changes on redeploy, the cache should be present + run!(%Q{git commit --allow-empty -m next}) + app.push! - # Redeploy with changed requirements file - run!(%Q{echo "" >> requirements.txt}) - run!(%Q{echo "flask" >> requirements.txt}) - run!(%Q{git add . ; git commit --allow-empty -m next}) - app.push! - - # Check the cache to have cleared - expect(app.output).to match("Requirements file has been changed, clearing cached dependencies") - - # What should not happen when the requirements file is changed - expect(app.output).to_not match("No dependencies found, preparing to install") - expect(app.output).to_not match("No change in requirements detected, installing from cache") - - # let the previous build finish - sleep(5) - - run!(%Q{git commit --allow-empty -m next}) - app.push! + expect(app.output).to match("No change in requirements detected, installing from cache") + expect(app.output).to_not match("Requirements file has been changed, clearing cached dependencies") + expect(app.output).to_not match("No dependencies found, preparing to install") + end + end + end - # With no changes on redeploy, the cache should - expect(app.output).to match("No change in requirements detected, installing from cache") + describe "python versions" do + let(:stack) { ENV["HEROKU_TEST_STACK"] || DEFAULT_STACK } + it "works with 3.7.6" do + version = "3.7.6" + before_deploy = -> { run!(%Q{echo "python-#{version}" >> runtime.txt}) } + Hatchet::Runner.new("python_default", before_deploy: before_deploy, stack: stack).deploy do |app| + expect(app.run('python -V')).to match(version) + end + end - # With no changes on redeploy, the cache should not - expect(app.output).to_not match("Requirements file has been changed, clearing cached dependencies") - expect(app.output).to_not match("No dependencies found, preparing to install") + it "works with 3.8.2" do + version = "3.8.2" + before_deploy = -> { run!(%Q{echo "python-#{version}" >> runtime.txt}) } + Hatchet::Runner.new("python_default", before_deploy: before_deploy, stack: stack).deploy do |app| + expect(app.run('python -V')).to match(version) + end + end - expect(app.run('python -V')).to match(version) - end + it "fails with a bad version" do + version = "3.8.2.lol" + before_deploy = -> { run!(%Q{echo "python-#{version}" >> runtime.txt}) } + Hatchet::Runner.new("python_default", before_deploy: before_deploy, stack: stack, allow_failure: true).deploy do |app| + expect(app.output).to match("not available for this stack") end end end + + it "getting started app has no relative paths" do + buildpacks = [ + :default, + "https://github.com/sharpstone/force_absolute_paths_buildpack" + ] + Hatchet::Runner.new("python-getting-started", buildpacks: buildpacks).deploy do |app| + # Deploy works + end + end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index a53d735d5..34f28b3cf 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,4 +1,4 @@ -ENV['HATCHET_BUILDPACK_BASE'] = 'https://github.com/' + ENV['TRAVIS_REPO_SLUG'] + '.git' +ENV['HATCHET_BUILDPACK_BASE'] = 'https://github.com/heroku/heroku-buildpack-python.git' require 'rspec/core' require 'rspec/retry' @@ -32,8 +32,3 @@ def run!(cmd) raise "Error running command #{cmd} with output: #{out}" unless $?.success? return out end - -def init_app(app, stack=DEFAULT_STACK) - app.setup! - app.platform_api.app.update(app.name, {"build_stack" => ENV["HEROKU_TEST_STACK"] || stack}) -end \ No newline at end of file