Skip to content

Commit

Permalink
Merge pull request #8312 from dependabot/deivid-rodriguez/pipenv-upgrade
Browse files Browse the repository at this point in the history 
Replace `pipenv lock` with `pipenv upgrade`
  • Loading branch information
@deivid-rodriguez
deivid-rodriguez committed Nov 2, 2023
2 parents 9600565 + eec1c1a commit c947610
Show file tree
Hide file tree
Showing 12 changed files with 171 additions and 429 deletions.
3 changes: 2 additions & 1 deletion python/lib/dependabot/python/file_parser/pipfile_files_parser.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,8 @@ def pipfile_dependencies
source: nil,
groups: [group]
}],
package_manager: "pip"
package_manager: "pip",
metadata: { original_name: dep_name }
)
end
end
Expand Down
90 changes: 16 additions & 74 deletions python/lib/dependabot/python/file_updater/pipfile_file_updater.rb
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
# typed: true
# frozen_string_literal: true

require "toml-rb"
require "open3"
require "dependabot/dependency"
require "dependabot/python/requirement_parser"
Expand All @@ -10,7 +9,7 @@
require "dependabot/python/language_version_manager"
require "dependabot/shared_helpers"
require "dependabot/python/native_helpers"
require "dependabot/python/name_normaliser"
require "dependabot/python/pipenv_runner"

module Dependabot
module Python
Expand Down Expand Up @@ -84,7 +83,6 @@ def generated_requirements_files(type)
return [] unless lockfile

pipfile_lock_deps = parsed_lockfile[type]&.keys&.sort || []
pipfile_lock_deps = pipfile_lock_deps.map { |n| normalise(n) }
return [] unless pipfile_lock_deps.any?

regex = RequirementParser::INSTALL_REQ_WITH_REQUIREMENT
Expand All @@ -95,7 +93,7 @@ def generated_requirements_files(type)
requirements_files.select do |req_file|
deps = []
req_file.content.scan(regex) { deps << Regexp.last_match }
deps = deps.map { |m| normalise(m[:name]) }
deps = deps.map { |m| m[:name] }
deps.sort == pipfile_lock_deps
end
end
Expand Down Expand Up @@ -130,61 +128,17 @@ def updated_dev_req_content

def prepared_pipfile_content
content = updated_pipfile_content
content = freeze_other_dependencies(content)
content = freeze_dependencies_being_updated(content)
content = add_private_sources(content)
content = update_python_requirement(content)
content
end

def freeze_other_dependencies(pipfile_content)
PipfilePreparer
.new(pipfile_content: pipfile_content, lockfile: lockfile)
.freeze_top_level_dependencies_except(dependencies)
end

def update_python_requirement(pipfile_content)
PipfilePreparer
.new(pipfile_content: pipfile_content)
.update_python_requirement(language_version_manager.python_major_minor)
end

# rubocop:disable Metrics/PerceivedComplexity
def freeze_dependencies_being_updated(pipfile_content)
pipfile_object = TomlRB.parse(pipfile_content)

dependencies.each do |dep|
DEPENDENCY_TYPES.each do |type|
names = pipfile_object[type]&.keys || []
pkg_name = names.find { |nm| normalise(nm) == dep.name }
next unless pkg_name || subdep_type?(type)

pkg_name ||= dependency.name
if pipfile_object[type][pkg_name].is_a?(Hash)
pipfile_object[type][pkg_name]["version"] =
"==#{dep.version}"
else
pipfile_object[type][pkg_name] = "==#{dep.version}"
end
end
end

TomlRB.dump(pipfile_object)
end
# rubocop:enable Metrics/PerceivedComplexity

def subdep_type?(type)
return false if dependency.top_level?

lockfile_type = Python::FileParser::DEPENDENCY_GROUP_KEYS
.find { |i| i.fetch(:pipfile) == type }
.fetch(:lockfile)

JSON.parse(lockfile.content)
.fetch(lockfile_type, {})
.keys.any? { |k| normalise(k) == dependency.name }
end

def add_private_sources(pipfile_content)
PipfilePreparer
.new(pipfile_content: pipfile_content)
Expand All @@ -198,9 +152,7 @@ def updated_generated_files
write_temporary_dependency_files(prepared_pipfile_content)
install_required_python

run_pipenv_command(
"pyenv exec pipenv lock"
)
pipenv_runner.run_upgrade("==#{dependency.version}")

result = { lockfile: File.read("Pipfile.lock") }
result[:lockfile] = post_process_lockfile(result[:lockfile])
Expand Down Expand Up @@ -246,17 +198,12 @@ def generate_updated_requirements_files
File.write("dev-req.txt", dev_req_content)
end

def run_command(command, env: {}, fingerprint: nil)
SharedHelpers.run_shell_command(command, env: env, fingerprint: fingerprint)
def run_command(command)
SharedHelpers.run_shell_command(command)
end

def run_pipenv_command(command, env: pipenv_env_variables)
run_command(
"pyenv local #{language_version_manager.python_major_minor}",
fingerprint: "pyenv local <python_major_minor>"
)

run_command(command, env: env)
def run_pipenv_command(command)
pipenv_runner.run(command)
end

def write_temporary_dependency_files(pipfile_content)
Expand Down Expand Up @@ -329,10 +276,6 @@ def updated_file(file:, content:)
updated_file
end

def normalise(name)
NameNormaliser.normalise(name)
end

def python_requirement_parser
@python_requirement_parser ||=
FileParser::PythonRequirementParser.new(
Expand All @@ -347,6 +290,15 @@ def language_version_manager
)
end

def pipenv_runner
@pipenv_runner ||=
PipenvRunner.new(
dependency: dependency,
lockfile: lockfile,
language_version_manager: language_version_manager
)
end

def parsed_lockfile
@parsed_lockfile ||= JSON.parse(lockfile.content)
end
Expand All @@ -370,16 +322,6 @@ def setup_cfg_files
def requirements_files
dependency_files.select { |f| f.name.end_with?(".txt") }
end

def pipenv_env_variables
{
"PIPENV_YES" => "true", # Install new Python ver if needed
"PIPENV_MAX_RETRIES" => "3", # Retry timeouts
"PIPENV_NOSPIN" => "1", # Don't pollute logs with spinner
"PIPENV_TIMEOUT" => "600", # Set install timeout to 10 minutes
"PIP_DEFAULT_TIMEOUT" => "60" # Set pip timeout to 1 minute
}
end
end
end
end
Expand Down
68 changes: 1 addition & 67 deletions python/lib/dependabot/python/file_updater/pipfile_preparer.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,13 @@
require "dependabot/python/file_parser"
require "dependabot/python/file_updater"
require "dependabot/python/authed_url_builder"
require "dependabot/python/name_normaliser"

module Dependabot
module Python
class FileUpdater
class PipfilePreparer
def initialize(pipfile_content:, lockfile: nil)
def initialize(pipfile_content:)
@pipfile_content = pipfile_content
@lockfile = lockfile
end

def replace_sources(credentials)
Expand All @@ -28,45 +26,6 @@ def replace_sources(credentials)
TomlRB.dump(pipfile_object)
end

def freeze_top_level_dependencies_except(dependencies)
return pipfile_content unless lockfile

pipfile_object = TomlRB.parse(pipfile_content)
excluded_names = dependencies.map(&:name)

Python::FileParser::DEPENDENCY_GROUP_KEYS.each do |keys|
next unless pipfile_object[keys[:pipfile]]

pipfile_object.fetch(keys[:pipfile]).each do |dep_name, _|
next if excluded_names.include?(normalise(dep_name))

freeze_dependency(dep_name, pipfile_object, keys)
end
end

TomlRB.dump(pipfile_object)
end

def freeze_dependency(dep_name, pipfile_object, keys)
locked_version = version_from_lockfile(
keys[:lockfile],
normalise(dep_name)
)
locked_ref = ref_from_lockfile(
keys[:lockfile],
normalise(dep_name)
)

pipfile_req = pipfile_object[keys[:pipfile]][dep_name]
if pipfile_req.is_a?(Hash) && locked_version
pipfile_req["version"] = "==#{locked_version}"
elsif pipfile_req.is_a?(Hash) && locked_ref && !pipfile_req["ref"]
pipfile_req["ref"] = locked_ref
elsif locked_version
pipfile_object[keys[:pipfile]][dep_name] = "==#{locked_version}"
end
end

def update_python_requirement(requirement)
pipfile_object = TomlRB.parse(pipfile_content)

Expand All @@ -84,31 +43,6 @@ def update_python_requirement(requirement)

attr_reader :pipfile_content, :lockfile

def version_from_lockfile(dep_type, dep_name)
details = parsed_lockfile.dig(dep_type, normalise(dep_name))

case details
when String then details.gsub(/^==/, "")
when Hash then details["version"]&.gsub(/^==/, "")
end
end

def ref_from_lockfile(dep_type, dep_name)
details = parsed_lockfile.dig(dep_type, normalise(dep_name))

case details
when Hash then details["ref"]
end
end

def parsed_lockfile
@parsed_lockfile ||= JSON.parse(lockfile.content)
end

def normalise(name)
NameNormaliser.normalise(name)
end

def pipfile_sources
@pipfile_sources ||= TomlRB.parse(pipfile_content).fetch("source", [])
end
Expand Down
82 changes: 82 additions & 0 deletions python/lib/dependabot/python/pipenv_runner.rb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
# typed: true
# frozen_string_literal: true

require "dependabot/shared_helpers"
require "dependabot/python/file_parser"
require "json"

module Dependabot
module Python
class PipenvRunner
def initialize(dependency:, lockfile:, language_version_manager:)
@dependency = dependency
@lockfile = lockfile
@language_version_manager = language_version_manager
end

def run_upgrade(constraint)
command = "pyenv exec pipenv upgrade #{dependency_name}#{constraint}"
command << " --dev" if lockfile_section == "develop"

run(command)
end

def run_upgrade_and_fetch_version(constraint)
run_upgrade(constraint)

updated_lockfile = JSON.parse(File.read("Pipfile.lock"))

fetch_version_from_parsed_lockfile(updated_lockfile)
end

def run(command)
run_command(
"pyenv local #{language_version_manager.python_major_minor}",
fingerprint: "pyenv local <python_major_minor>"
)

run_command(command)
end

private

attr_reader :dependency, :lockfile, :language_version_manager

def fetch_version_from_parsed_lockfile(updated_lockfile)
deps = updated_lockfile[lockfile_section] || {}

deps.dig(dependency_name, "version")
&.gsub(/^==/, "")
end

def run_command(command, fingerprint: nil)
SharedHelpers.run_shell_command(command, env: pipenv_env_variables, fingerprint: fingerprint)
end

def lockfile_section
if dependency.requirements.any?
dependency.requirements.first[:groups].first
else
Python::FileParser::DEPENDENCY_GROUP_KEYS.each do |keys|
section = keys.fetch(:lockfile)
return section if JSON.parse(lockfile.content)[section].keys.any?(dependency_name)
end
end
end

def dependency_name
dependency.metadata[:original_name] || dependency.name
end

def pipenv_env_variables
{
"PIPENV_YES" => "true", # Install new Python ver if needed
"PIPENV_MAX_RETRIES" => "3", # Retry timeouts
"PIPENV_NOSPIN" => "1", # Don't pollute logs with spinner
"PIPENV_TIMEOUT" => "600", # Set install timeout to 10 minutes
"PIP_DEFAULT_TIMEOUT" => "60" # Set pip timeout to 1 minute
}
end
end
end
end
6 changes: 3 additions & 3 deletions python/lib/dependabot/python/update_checker.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,11 +222,11 @@ def unlocked_requirement_string
return lower_bound_req if latest_version.nil?
return lower_bound_req unless Python::Version.correct?(latest_version)

lower_bound_req + ", <= #{latest_version}"
lower_bound_req + ",<=#{latest_version}"
end

def updated_version_req_lower_bound
return ">= #{dependency.version}" if dependency.version
return ">=#{dependency.version}" if dependency.version

version_for_requirement =
requirements.filter_map { |r| r[:requirement] }
Expand All @@ -236,7 +236,7 @@ def updated_version_req_lower_bound
.select { |version| Gem::Version.correct?(version) }
.max_by { |version| Gem::Version.new(version) }

">= #{version_for_requirement || 0}"
">=#{version_for_requirement || 0}"
end

def fetch_latest_version
Expand Down
Loading

0 comments on commit c947610

Please sign in to comment.