Skip to content

Commit

Permalink
Always use nuget to install python on Windows
Browse files Browse the repository at this point in the history
  • Loading branch information
@mayeut
mayeut committed Nov 3, 2019
1 parent a6164f7 commit a1c095a
Show file tree
Hide file tree
Showing 3 changed files with 57 additions and 89 deletions.
17 changes: 5 additions & 12 deletions azure-pipelines.yml
View file
Original file line number Diff line number Diff line change
@@ -1,31 +1,24 @@
jobs:
- job: linux
pool: {vmImage: 'Ubuntu-16.04'}
steps:
pool: {vmImage: 'Ubuntu-18.04'}
steps:
- task: UsePythonVersion@0
- bash: |
python -m pip install -r requirements-dev.txt
python ./bin/run_tests.py
- job: macos
pool: {vmImage: 'macOS-10.13'}
steps:
steps:
- task: UsePythonVersion@0
- bash: |
python -m pip install -r requirements-dev.txt
python ./bin/run_tests.py
- job: windows
pool: {vmImage: 'vs2017-win2016'}
steps:
- {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x86}}
- {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x64}}
- {task: UsePythonVersion@0, inputs: {versionSpec: '3.5', architecture: x86}}
- {task: UsePythonVersion@0, inputs: {versionSpec: '3.5', architecture: x64}}
- {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x86}}
- {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x64}}
- {task: UsePythonVersion@0, inputs: {versionSpec: '3.7', architecture: x86}}
- {task: UsePythonVersion@0, inputs: {versionSpec: '3.7', architecture: x64}}
steps:
- task: UsePythonVersion@0
- script: choco install vcpython27 -f -y
displayName: Install Visual C++ for Python 2.7
- bash: |
Expand Down
115 changes: 45 additions & 70 deletions cibuildwheel/windows.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,89 +8,71 @@
except ImportError:
from pipes import quote as shlex_quote

try:
from urllib2 import urlopen
except ImportError:
from urllib.request import urlopen

from .util import prepare_command, get_build_verbosity_extra_flags


IS_RUNNING_ON_AZURE = os.path.exists('C:\\hostedtoolcache')
IS_RUNNING_ON_TRAVIS = os.environ.get('TRAVIS_OS_NAME') == 'windows'


def get_python_path(config):
if IS_RUNNING_ON_AZURE:
# We can't hard-code the paths because on Azure, we don't know which
# bugfix release of Python we are getting so we need to check which
# ones exist. We just use the first one that is found since there should
# only be one.
path_pattern = 'C:\\hostedtoolcache\\windows\\Python\\{version}\\{arch}'.format(
version=config.version.replace('x', '*'),
arch='x86' if config.arch == '32' else 'x64'
)
try:
return glob(path_pattern)[0]
except IndexError:
raise Exception('Could not find a Python install at ' + path_pattern)
elif IS_RUNNING_ON_TRAVIS:
if config.version == "3.4.x":
return config.path
else:
nuget_args = get_nuget_args(config)
return os.path.join(nuget_args[-1], nuget_args[0] + "." + config.nuget_version, "tools")
else:
# Assume we're running on AppVeyor
major, minor = config.version.split('.')[:2]
return 'C:\\Python{major}{minor}{arch}'.format(
major=major,
minor=minor,
arch = '-x64' if config.arch == '64' else ''
)
nuget_args = get_nuget_args(config)
return os.path.join(nuget_args[-1], nuget_args[0] + "." + config.version, "tools")


def get_nuget_args(configuration):
if configuration.nuget_version is None:
return None
python_name = "python" if configuration.version[0] == '3' else "python2"
if configuration.arch == "32":
python_name = python_name + "x86"
return [python_name, "-Version", configuration.nuget_version, "-OutputDirectory", "C:/python"]
return [python_name, "-Version", configuration.version, "-OutputDirectory", "C:/python"]

def get_python_configurations(build_selector):
PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier', 'path', "nuget_version"])
PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier'])
python_configurations = [
PythonConfiguration(version='2.7.x', arch="32", identifier='cp27-win32', path='C:\\Python27', nuget_version="2.7.16"),
PythonConfiguration(version='2.7.x', arch="64", identifier='cp27-win_amd64', path='C:\\Python27-x64', nuget_version="2.7.16"),
PythonConfiguration(version='3.4.x', arch="32", identifier='cp34-win32', path='C:\\Python34', nuget_version=None),
PythonConfiguration(version='3.4.x', arch="64", identifier='cp34-win_amd64', path='C:\\Python34-x64', nuget_version=None),
PythonConfiguration(version='3.5.x', arch="32", identifier='cp35-win32', path='C:\\Python35', nuget_version="3.5.4"),
PythonConfiguration(version='3.5.x', arch="64", identifier='cp35-win_amd64', path='C:\\Python35-x64', nuget_version="3.5.4"),
PythonConfiguration(version='3.6.x', arch="32", identifier='cp36-win32', path='C:\\Python36', nuget_version="3.6.8"),
PythonConfiguration(version='3.6.x', arch="64", identifier='cp36-win_amd64', path='C:\\Python36-x64', nuget_version="3.6.8"),
PythonConfiguration(version='3.7.x', arch="32", identifier='cp37-win32', path='C:\\Python37', nuget_version="3.7.4"),
PythonConfiguration(version='3.7.x', arch="64", identifier='cp37-win_amd64', path='C:\\Python37-x64', nuget_version="3.7.4")
PythonConfiguration(version='2.7.17', arch="32", identifier='cp27-win32'),
PythonConfiguration(version='2.7.17', arch="64", identifier='cp27-win_amd64'),
PythonConfiguration(version='3.5.4', arch="32", identifier='cp35-win32'),
PythonConfiguration(version='3.5.4', arch="64", identifier='cp35-win_amd64'),
PythonConfiguration(version='3.6.8', arch="32", identifier='cp36-win32'),
PythonConfiguration(version='3.6.8', arch="64", identifier='cp36-win_amd64'),
PythonConfiguration(version='3.7.5', arch="32", identifier='cp37-win32'),
PythonConfiguration(version='3.7.5', arch="64", identifier='cp37-win_amd64'),
PythonConfiguration(version='3.8.0', arch="32", identifier='cp38-win32'),
PythonConfiguration(version='3.8.0', arch="64", identifier='cp38-win_amd64'),
]

if IS_RUNNING_ON_AZURE:
# Python 3.4 isn't supported on Azure.
# See https://github.com/Microsoft/azure-pipelines-tasks/issues/9674
python_configurations = [c for c in python_configurations if c.version != '3.4.x']

if IS_RUNNING_ON_TRAVIS:
# cannot install VCForPython27.msi which is needed for compiling C software
# try with (and similar): msiexec /i VCForPython27.msi ALLUSERS=1 ACCEPT=YES /passive
# no easy and stable way fo installing python 3.4
python_configurations = [c for c in python_configurations if c.version != '2.7.x' and c.version != '3.4.x']
python_configurations = [c for c in python_configurations if not c.version.startswith('2.7.')]

# skip builds as required
python_configurations = [c for c in python_configurations if build_selector(c.identifier)]

return python_configurations



def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, environment):
def simple_shell(args, env=None, cwd=None):
print('+ ' + ' '.join(args))
args = ['cmd', '/E:ON', '/V:ON', '/C'] + args
return subprocess.check_call(' '.join(args), env=env, cwd=cwd)
def download(url, dest):
print('+ Download ' + url + ' to ' + dest)
response = urlopen(url)
try:
with open(dest, 'wb') as file:
file.write(response.read())
finally:
response.close()
if IS_RUNNING_ON_AZURE or IS_RUNNING_ON_TRAVIS:
def shell(args, env=None, cwd=None):
print('+ ' + ' '.join(args))
args = ['cmd', '/E:ON', '/V:ON', '/C'] + args
return subprocess.check_call(' '.join(args), env=env, cwd=cwd)
shell = simple_shell
else:
run_with_env = os.path.abspath(os.path.join(os.path.dirname(__file__), 'resources', 'appveyor_run_with_env.cmd'))

Expand All @@ -106,21 +88,19 @@ def shell(args, env=None, cwd=None):
temp_dir = tempfile.mkdtemp(prefix='cibuildwheel')
built_wheel_dir = os.path.join(temp_dir, 'built_wheel')

if IS_RUNNING_ON_TRAVIS:
# instal nuget as best way for provide python
shell(["choco", "install", "nuget.commandline"])
# get pip fo this installation which not have.
get_pip_url = 'https://bootstrap.pypa.io/get-pip.py'
get_pip_script = 'C:\\get-pip.py'
shell(['curl', '-L', '-o', get_pip_script, get_pip_url])
# install nuget as best way to provide python
nuget = 'C:\\nuget.exe'
download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget)
# get pip fo this installation which not have.
get_pip_script = 'C:\\get-pip.py'
download('https://bootstrap.pypa.io/get-pip.py', get_pip_script)

python_configurations = get_python_configurations(build_selector)
for config in python_configurations:
config_python_path = get_python_path(config)
if IS_RUNNING_ON_TRAVIS and config.nuget_version is not None and not os.path.exists(config_python_path):
shell(["nuget", "install"] + get_nuget_args(config))
if not os.path.exists(os.path.join(config_python_path, 'Scripts', 'pip.exe')):
shell([os.path.join(config_python_path, 'python.exe'), get_pip_script ])
simple_shell([nuget, "install"] + get_nuget_args(config))
if not os.path.exists(os.path.join(config_python_path, 'Scripts', 'pip.exe')):
simple_shell([os.path.join(config_python_path, 'python.exe'), get_pip_script ])

# check python & pip exist for this configuration
assert os.path.exists(os.path.join(config_python_path, 'python.exe'))
Expand All @@ -147,12 +127,7 @@ def shell(args, env=None, cwd=None):
shell(['python', '-c', '"import struct; print(struct.calcsize(\'P\') * 8)\"'], env=env)

# prepare the Python environment
if config.version == "3.4.x":
shell(['python', '-m', 'pip', 'install', 'pip==19.1.1'],
env=env)
else:
shell(['python', '-m', 'pip', 'install', '--upgrade', 'pip'],
env=env)
shell(['python', '-m', 'pip', 'install', '--upgrade', 'pip'], env=env)
shell(['pip', 'install', '--upgrade', 'setuptools'], env=env)
shell(['pip', 'install', 'wheel'], env=env)

Expand Down
14 changes: 7 additions & 7 deletions test/shared/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,14 @@ def cibuildwheel_get_build_identifiers(project_path, env=None):

def cibuildwheel_run(project_path, env=None, add_env=None):
'''
Runs cibuildwheel as a subprocess, building the project at project_path.
Runs cibuildwheel as a subprocess, building the project at project_path.
Uses the current Python interpreter.
Configure settings using env.
'''
if env is None:
env = os.environ.copy()

if add_env is not None:
env.update(add_env)

Expand Down Expand Up @@ -77,15 +77,15 @@ def expected_wheels(package_name, package_version):
elif platform == 'windows':
templates = [
'{package_name}-{package_version}-cp27-cp27m-win32.whl',
'{package_name}-{package_version}-cp34-cp34m-win32.whl',
'{package_name}-{package_version}-cp35-cp35m-win32.whl',
'{package_name}-{package_version}-cp36-cp36m-win32.whl',
'{package_name}-{package_version}-cp37-cp37m-win32.whl',
'{package_name}-{package_version}-cp38-cp38-win32.whl',
'{package_name}-{package_version}-cp27-cp27m-win_amd64.whl',
'{package_name}-{package_version}-cp34-cp34m-win_amd64.whl',
'{package_name}-{package_version}-cp35-cp35m-win_amd64.whl',
'{package_name}-{package_version}-cp36-cp36m-win_amd64.whl',
'{package_name}-{package_version}-cp37-cp37m-win_amd64.whl',
'{package_name}-{package_version}-cp38-cp38-win_amd64.whl',
]
elif platform == 'macos':
templates = [
Expand All @@ -97,14 +97,14 @@ def expected_wheels(package_name, package_version):
]
else:
raise Exception('unsupported platform')

if IS_WINDOWS_RUNNING_ON_AZURE:
# Python 3.4 isn't supported on Azure.
templates = [t for t in templates if '-cp34-' not in t]
if IS_WINDOWS_RUNNING_ON_TRAVIS:
# Python 2.7 and 3.4 isn't supported on Travis.
templates = [t for t in templates if '-cp27-' not in t and '-cp34-' not in t]

return [filename.format(package_name=package_name, package_version=package_version)
for filename in templates]

Expand Down

0 comments on commit a1c095a

Please sign in to comment.