Merge pull request #3211 from dependabot/feelepxyz/bundler1-native-specs

Bundler: Run v1 native helpers with bundler v1

.github/workflows/ci.yml

@@ -102,6 +102,11 @@ jobs:
         run: |
           docker run --rm "$CORE_CI_IMAGE" bash -c "cd /opt/npm_and_yarn && npm run lint"
           docker run --rm "$CORE_CI_IMAGE" bash -c "cd /opt/npm_and_yarn && npm test"
+      - name: Run bundler v1 native helper specs
+        if: matrix.suite == 'bundler'
+        run: |
+          docker run --rm "$CORE_CI_IMAGE" bash -c \
+            "cd /home/dependabot/dependabot-core/bundler/helpers/v1 && BUNDLER_VERSION=1 bundle install && BUNDLER_VERSION=1 bundle exec rspec spec"
       - name: Run ${{ matrix.suite }} tests with rspec
         run: |
           docker run --env "CI=true" --env "DEPENDABOT_TEST_ACCESS_TOKEN=$DEPENDABOT_TEST_ACCESS_TOKEN" --rm "$CORE_CI_IMAGE" bash -c \

bundler/helpers/v1/Gemfile

@@ -2,6 +2,11 @@
 
 source "https://rubygems.org"
 
-# NOTE: This is intentionally left blank as it's currently only used to force
-# bundler to use v1 when executing native helpers by pointing the BUNDLE_GEMFILE
-# env to this Gemfile in Dependabot::Bundler::NativeHelpers
+# NOTE: Used to run native helper specs
+group :test do
+  gem "byebug", "11.1.3"
+  gem "rspec", "3.10.0"
+  gem "rspec-its", "1.3.0"
+  gem "vcr", "6.0.0"
+  gem "webmock", "3.12.1"
+end

bundler/helpers/v1/build

@@ -20,5 +20,5 @@ cp -r \
 cd "$install_dir"
 
 # NOTE: Sets `BUNDLED WITH` to match the installed v1 version in Gemfile.lock
-# forcing specs and native helpers to run with the same version
-BUNDLER_VERSION=1 bundle install
+# forcing native helpers to run with the same version
+BUNDLER_VERSION=1 bundle install --without test

bundler/spec/native_helpers/functions/conflicting_dependency_resolver_spec.rb → bundler/helpers/v1/spec/functions/conflicting_dependency_resolver_spec.rb

@@ -10,15 +10,14 @@
     described_class.new(
       dependency_name: dependency_name,
       target_version: target_version,
-      lockfile_name: lockfile_name
+      lockfile_name: "Gemfile.lock"
     )
   end
 
   let(:dependency_name) { "dummy-pkg-a" }
   let(:target_version) { "2.0.0" }
 
-  let(:gemfile_fixture_name) { "blocked_by_subdep" }
-  let(:lockfile_fixture_name) { "blocked_by_subdep.lock" }
+  let(:project_name) { "blocked_by_subdep" }
 
   describe "#conflicting_dependencies" do
     subject(:conflicting_dependencies) do
@@ -37,8 +36,7 @@
     end
 
     context "for nested transitive dependencies" do
-      let(:gemfile_fixture_name) { "transitive_blocking" }
-      let(:lockfile_fixture_name) { "transitive_blocking.lock" }
+      let(:project_name) { "transitive_blocking" }
       let(:dependency_name) { "activesupport" }
       let(:target_version) { "6.0.0" }
 
@@ -96,8 +94,7 @@
       let(:dependency_name) { "activesupport" }
       let(:current_version) { "5.0.0" }
       let(:target_version) { "6.0.0" }
-      let(:gemfile_fixture_name) { "multiple_blocking" }
-      let(:lockfile_fixture_name) { "multiple_blocking.lock" }
+      let(:project_name) { "multiple_blocking" }
 
       it "returns all of the blocking dependencies" do
         expect(conflicting_dependencies).to match_array(

bundler/spec/native_helpers/functions/dependency_source_spec.rb → bundler/helpers/v1/spec/functions/dependency_source_spec.rb

@@ -8,14 +8,14 @@
 
   let(:dependency_source) do
     described_class.new(
-      gemfile_name: gemfile_name,
+      gemfile_name: "Gemfile",
       dependency_name: dependency_name
     )
   end
 
   let(:dependency_name) { "business" }
 
-  let(:gemfile_fixture_name) { "specified_source" }
+  let(:project_name) { "specified_source_no_lockfile" }
   let(:registry_url) { "https://repo.fury.io/greysteil/" }
   let(:gemfury_business_url) do
     "https://repo.fury.io/greysteil/api/v1/dependencies?gems=business"
@@ -47,7 +47,7 @@
     end
 
     context "specified as the default source" do
-      let(:gemfile_fixture_name) { "specified_default_source" }
+      let(:project_name) { "specified_default_source_no_lockfile" }
 
       it "returns all versions from the private source" do
         is_expected.to eq([
@@ -152,7 +152,7 @@
     end
 
     context "that only implements the old Bundler index format..." do
-      let(:gemfile_fixture_name) { "sidekiq_pro" }
+      let(:project_name) { "sidekiq_pro" }
       let(:dependency_name) { "sidekiq-pro" }
       let(:registry_url) { "https://gems.contribsys.com/" }
 

bundler/spec/native_helpers/functions/file_parser_spec.rb → bundler/helpers/v1/spec/functions/file_parser_spec.rb

@@ -12,8 +12,7 @@
     )
   end
 
-  let(:gemfile_fixture_name) { "Gemfile" }
-  let(:lockfile_fixture_name) { "Gemfile.lock" }
+  let(:project_name) { "gemfile" }
 
   describe "#parsed_gemfile" do
     subject(:parsed_gemfile) do

bundler/spec/native_helpers/functions/version_resolver_spec.rb → bundler/helpers/v1/spec/functions/version_resolver_spec.rb

@@ -3,8 +3,6 @@
 require "native_spec_helper"
 require "shared_contexts"
 
-require "dependabot/dependency"
-
 RSpec.describe Functions::VersionResolver do
   include_context "in a temporary bundler directory"
   include_context "stub rubygems compact index"
@@ -13,8 +11,8 @@
     described_class.new(
       dependency_name: dependency_name,
       dependency_requirements: dependency_requirements,
-      gemfile_name: gemfile_name,
-      lockfile_name: lockfile_name
+      gemfile_name: "Gemfile",
+      lockfile_name: "Gemfile.lock"
     )
   end
 
@@ -37,8 +35,7 @@
       in_tmp_folder { version_resolver.version_details }
     end
 
-    let(:gemfile_fixture_name) { "Gemfile" }
-    let(:lockfile_fixture_name) { "Gemfile.lock" }
+    let(:project_name) { "gemfile" }
     let(:requirement_string) { " >= 0" }
 
     its([:version]) { is_expected.to eq(Gem::Version.new("1.4.0")) }
@@ -47,8 +44,7 @@
     context "with a private gemserver source" do
       include_context "stub rubygems compact index"
 
-      let(:gemfile_fixture_name) { "specified_source" }
-      let(:lockfile_fixture_name) { "specified_source.lock" }
+      let(:project_name) { "specified_source" }
       let(:requirement_string) { ">= 0" }
 
       before do
@@ -72,8 +68,7 @@
     end
 
     context "with a git source" do
-      let(:gemfile_fixture_name) { "git_source" }
-      let(:lockfile_fixture_name) { "git_source.lock" }
+      let(:project_name) { "git_source" }
 
       its([:version]) { is_expected.to eq(Gem::Version.new("1.6.0")) }
       its([:fetcher]) { is_expected.to be_nil }

bundler/helpers/v1/spec/native_spec_helper.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require "rspec/its"
+require "webmock/rspec"
+require "byebug"
+
+$LOAD_PATH.unshift(File.expand_path("../lib", __dir__))
+$LOAD_PATH.unshift(File.expand_path("../monkey_patches", __dir__))
+
+# Bundler monkey patches
+require "definition_ruby_version_patch"
+require "definition_bundler_version_patch"
+require "git_source_patch"
+
+require "functions"
+
+RSpec.configure do |config|
+  config.color = true
+  config.order = :rand
+  config.mock_with(:rspec) { |mocks| mocks.verify_partial_doubles = true }
+  config.raise_errors_for_deprecations!
+end
+
+# Duplicated in lib/dependabot/bundler/file_updater/lockfile_updater.rb
+# TODO: Stop sanitizing the lockfile once we have bundler 2 installed
+LOCKFILE_ENDING = /(?<ending>\s*(?:RUBY VERSION|BUNDLED WITH).*)/m.freeze
+
+def project_dependency_files(project)
+  project_path = File.expand_path(File.join("../../spec/fixtures/projects/bundler1", project))
+  Dir.chdir(project_path) do
+    # NOTE: Include dotfiles (e.g. .npmrc)
+    files = Dir.glob("**/*", File::FNM_DOTMATCH)
+    files = files.select { |f| File.file?(f) }
+    files.map do |filename|
+      content = File.read(filename)
+      if filename == "Gemfile.lock"
+        content = content.gsub(LOCKFILE_ENDING, "")
+      end
+      {
+        name: filename,
+        content: content
+      }
+    end
+  end
+end
+
+def fixture(*name)
+  File.read(File.join("../../spec/fixtures", File.join(*name)))
+end

bundler/helpers/v1/spec/shared_contexts.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "bundler/compact_index_client"
+require "bundler/compact_index_client/updater"
+
+TMP_DIR_PATH = File.expand_path("../tmp", __dir__)
+
+RSpec.shared_context "in a temporary bundler directory" do
+  let(:project_name) { "gemfile" }
+
+  let(:tmp_path) do
+    Dir.mkdir(TMP_DIR_PATH) unless Dir.exist?(TMP_DIR_PATH)
+    dir = Dir.mktmpdir("native_helper_spec_", TMP_DIR_PATH)
+    Pathname.new(dir).expand_path
+  end
+
+  before do
+    project_dependency_files(project_name).each do |file|
+      File.write(File.join(tmp_path, file[:name]), file[:content])
+    end
+  end
+
+  def in_tmp_folder(&block)
+    Dir.chdir(tmp_path, &block)
+  end
+end
+
+RSpec.shared_context "without caching rubygems" do
+  before do
+    # Stub Bundler to stop it using a cached versions of Rubygems
+    allow_any_instance_of(Bundler::CompactIndexClient::Updater).
+      to receive(:etag_for).and_return("")
+  end
+end
+
+RSpec.shared_context "stub rubygems compact index" do
+  include_context "without caching rubygems"
+
+  before do
+    # Stub the Rubygems index
+    stub_request(:get, "https://index.rubygems.org/versions").
+      to_return(
+        status: 200,
+        body: fixture("ruby", "rubygems_responses", "index")
+      )
+
+    # Stub the Rubygems response for each dependency we have a fixture for
+    fixtures =
+      Dir[File.join("../../spec", "fixtures", "ruby", "rubygems_responses", "info-*")]
+    fixtures.each do |path|
+      dep_name = path.split("/").last.gsub("info-", "")
+      stub_request(:get, "https://index.rubygems.org/info/#{dep_name}").
+        to_return(
+          status: 200,
+          body: fixture("ruby", "rubygems_responses", "info-#{dep_name}")
+        )
+    end
+  end
+end

bundler/spec/fixtures/projects/bundler1/multiple_blocking/Gemfile

@@ -0,0 +1,5 @@
+source "https://rubygems.org"
+
+gem "activesupport", "5.0.0"
+gem "actionview", "5.0.0"
+gem "actionmailer", "5.0.0"

bundler/spec/fixtures/projects/bundler1/multiple_blocking/Gemfile.lock

@@ -0,0 +1,70 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    actionmailer (5.0.0)
+      actionpack (= 5.0.0)
+      actionview (= 5.0.0)
+      activejob (= 5.0.0)
+      mail (~> 2.5, >= 2.5.4)
+      rails-dom-testing (~> 2.0)
+    actionpack (5.0.0)
+      actionview (= 5.0.0)
+      activesupport (= 5.0.0)
+      rack (~> 2.0)
+      rack-test (~> 0.6.3)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.0.2)
+    actionview (5.0.0)
+      activesupport (= 5.0.0)
+      builder (~> 3.1)
+      erubis (~> 2.7.0)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.0.2)
+    activejob (5.0.0)
+      activesupport (= 5.0.0)
+      globalid (>= 0.3.6)
+    activesupport (5.0.0)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (~> 0.7)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    builder (3.2.4)
+    concurrent-ruby (1.1.7)
+    crass (1.0.6)
+    erubis (2.7.0)
+    globalid (0.4.2)
+      activesupport (>= 4.2.0)
+    i18n (0.9.5)
+      concurrent-ruby (~> 1.0)
+    loofah (2.7.0)
+      crass (~> 1.0.2)
+      nokogiri (>= 1.5.9)
+    mail (2.7.1)
+      mini_mime (>= 0.1.1)
+    mini_mime (1.0.2)
+    mini_portile2 (2.4.0)
+    minitest (5.14.2)
+    nokogiri (1.10.10)
+      mini_portile2 (~> 2.4.0)
+    rack (2.2.3)
+    rack-test (0.6.3)
+      rack (>= 1.0)
+    rails-dom-testing (2.0.3)
+      activesupport (>= 4.2.0)
+      nokogiri (>= 1.6)
+    rails-html-sanitizer (1.3.0)
+      loofah (~> 2.3)
+    thread_safe (0.3.6)
+    tzinfo (1.2.7)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  actionmailer (= 5.0.0)
+  actionview (= 5.0.0)
+  activesupport (= 5.0.0)
+
+BUNDLED WITH
+   2.1.4

bundler/spec/fixtures/projects/bundler1/sidekiq_pro/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+source "https://username:password@gems.contribsys.com/" do
+  gem 'sidekiq-pro'
+end

bundler/spec/fixtures/projects/bundler1/sidekiq_pro/Gemfile.lock

@@ -0,0 +1,26 @@
+GEM
+  remote: https://rubygems.org/
+  remote: https://username:password@gems.contribsys.com/
+  specs:
+    concurrent-ruby (1.0.5)
+    connection_pool (2.2.1)
+    rack (1.4.7)
+    rack-protection (1.5.3)
+      rack
+    redis (3.3.3)
+    sidekiq (4.2.9)
+      concurrent-ruby (~> 1.0)
+      connection_pool (~> 2.2, >= 2.2.0)
+      rack-protection (>= 1.5.0)
+      redis (~> 3.2, >= 3.2.1)
+    sidekiq-pro (3.4.4)
+      sidekiq (>= 4.1.5)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  sidekiq-pro!
+
+BUNDLED WITH
+   1.15.3

bundler/spec/fixtures/projects/bundler1/specified_default_source_no_lockfile/Gemfile

@@ -0,0 +1,3 @@
+source 'https://SECRET_CODES@repo.fury.io/greysteil/'
+
+gem 'business'

bundler/spec/fixtures/projects/bundler1/specified_source_no_lockfile/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+gem 'statesman'
+
+source 'https://SECRET_CODES@repo.fury.io/greysteil/' do
+  gem 'business'
+end