Skip to content

Commit

Permalink
[ci] Switch Windows build script to build.py (#6993)
Browse files Browse the repository at this point in the history 
Issue: #6445 

### Brief Summary
  • Loading branch information
@feisuzhu
feisuzhu authored Jan 17, 2023
1 parent 7ca9a61 commit 437fe47
Show file tree
Hide file tree
Showing 8 changed files with 278 additions and 105 deletions.
149 changes: 115 additions & 34 deletions .github/workflows/scripts/build.py
View file
Original file line number Diff line number Diff line change
@@ -1,25 +1,76 @@
#!/usr/bin/env python3

# -- prioritized --
import ci_common # isort: skip, early initialization happens here

# -- stdlib --
import glob
import os
import platform
from pathlib import Path

# -- third party --
# -- own --
from ci_common.dep import download_dep
from ci_common.misc import banner, get_cache_home, is_manylinux2014
from ci_common.python import setup_python
from ci_common.misc import (banner, get_cache_home, is_manylinux2014,
path_prepend)
from ci_common.python import path_prepend, setup_python
from ci_common.sccache import setup_sccache
from ci_common.tinysh import Command, environ, git, sh
from ci_common.tinysh import Command, git, sh


# -- code --
@banner('Setup Clang')
def setup_clang(as_compiler=True) -> None:
'''
Setup Clang.
'''
u = platform.uname()
if u.system == 'Linux':
pass
elif (u.system, u.machine) == ('Windows', 'AMD64'):
out = get_cache_home() / 'clang-15'
url = 'https://github.com/python3kgae/taichi_assets/releases/download/llvm15_vs2022_clang/clang-15.0.0-win.zip'
download_dep(url, out)
clang = str(out / 'bin' / 'clang++.exe').replace('\\', '\\\\')
os.environ['TAICHI_CMAKE_ARGS'] += f' -DCLANG_EXECUTABLE={clang}'

if as_compiler:
os.environ['TAICHI_CMAKE_ARGS'] += (
f' -DCMAKE_CXX_COMPILER={clang}'
f' -DCMAKE_C_COMPILER={clang}')
else:
# TODO: unify all
pass


@banner('Setup MSVC')
def setup_msvc() -> None:
assert platform.system() == 'Windows'
os.environ['TAICHI_USE_MSBUILD'] = '1'

url = 'https://aka.ms/vs/17/release/vs_BuildTools.exe'
out = Path(
r'C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools')
download_dep(
url,
out,
args=[
'--passive',
'--wait',
'--norestart',
'--includeRecommended',
'--add',
'Microsoft.VisualStudio.Workload.VCTools',
# NOTE: We are using the custom built Clang++,
# so components below are not necessary anymore.
# '--add',
# 'Microsoft.VisualStudio.Component.VC.Llvm.Clang',
# '--add',
# 'Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Llvm.Clang',
# '--add',
# 'Microsoft.VisualStudio.Component.VC.Llvm.ClangToolset',
])


@banner('Setup LLVM')
def setup_llvm(env_out: dict) -> None:
def setup_llvm() -> None:
'''
Download and install LLVM.
'''
Expand All @@ -28,7 +79,7 @@ def setup_llvm(env_out: dict) -> None:
if 'AMDGPU_TEST' in os.environ:
# FIXME: AMDGPU bots are currently maintained separately,
# we should unify them with the rest of the bots.
env_out['LLVM_DIR'] = '/taichi-llvm-15'
os.environ['LLVM_DIR'] = '/taichi-llvm-15'
return
elif is_manylinux2014():
# FIXME: prebuilt llvm15 on ubuntu didn't work on manylinux2014 image of centos. Once that's fixed, remove this hack.
Expand All @@ -37,54 +88,82 @@ def setup_llvm(env_out: dict) -> None:
else:
out = get_cache_home() / 'llvm15'
url = 'https://github.com/taichi-dev/taichi_assets/releases/download/llvm15/taichi-llvm-15-linux.zip'
download_dep(url, out, strip=1)
elif (u.system, u.machine) == ('Darwin', 'arm64'):
out = get_cache_home() / 'llvm15-m1'
url = 'https://github.com/taichi-dev/taichi_assets/releases/download/llvm15/taichi-llvm-15-m1.zip'
download_dep(url, out, strip=1)
elif (u.system, u.machine) == ('Darwin', 'x86_64'):
out = get_cache_home() / 'llvm15-mac'
url = 'https://github.com/taichi-dev/taichi_assets/releases/download/llvm15/llvm-15-mac10.15.zip'
download_dep(url, out, strip=1)
elif (u.system, u.machine) == ('Windows', 'AMD64'):
out = get_cache_home() / 'llvm15'
url = 'https://github.com/python3kgae/taichi_assets/releases/download/llvm15_vs2019_clang/taichi-llvm-15.0.0-msvc2019.zip'
download_dep(url, out, strip=0)
else:
raise RuntimeError(f'Unsupported platform: {u.system} {u.machine}')

download_dep(url, out, strip=1)
env_out['LLVM_DIR'] = str(out)
path_prepend('PATH', out / 'bin')
os.environ['LLVM_DIR'] = str(out)


@banner('Setup Vulkan 1.3.236.0')
def setup_vulkan(env: dict):
def setup_vulkan():
u = platform.uname()
if u.system == "Linux":
url = 'https://sdk.lunarg.com/sdk/download/1.3.236.0/linux/vulkansdk-linux-x86_64-1.3.236.0.tar.gz'
prefix = get_cache_home() / 'vulkan-1.3.236.0'
download_dep(url, prefix, strip=1)
sdk = prefix / 'x86_64'
env['VULKAN_SDK'] = str(sdk)
env['PATH'] = str(sdk / "bin") + ':' + env["PATH"]
env['LD_LIBRARY_PATH'] = str(sdk / "lib") + ':' + env.get(
"LD_LIBRARY_PATH", "")
env['VK_LAYER_PATH'] = str(sdk / 'etc' / 'vulkan' / 'explicit_layer.d')
os.environ['VULKAN_SDK'] = str(sdk)
path_prepend('PATH', sdk / "bin")
path_prepend('LD_LIBRARY_PATH', sdk / 'lib')
os.environ['VK_LAYER_PATH'] = str(sdk / 'etc' / 'vulkan' /
'explicit_layer.d')
# elif (u.system, u.machine) == ("Darwin", "arm64"):
# elif (u.system, u.machine) == ("Darwin", "x86_64"):
# elif u.system == "Windows":
elif (u.system, u.machine) == ('Windows', 'AMD64'):
url = 'https://sdk.lunarg.com/sdk/download/1.3.236.0/windows/VulkanSDK-1.3.236.0-Installer.exe'
prefix = get_cache_home() / 'vulkan-1.3.236.0'
download_dep(
url,
prefix,
args=[
'--accept-licenses',
'--default-answer',
'--confirm-command',
'--root',
prefix,
'install',
'com.lunarg.vulkan.sdl2',
'com.lunarg.vulkan.glm',
'com.lunarg.vulkan.volk',
'com.lunarg.vulkan.vma',
# 'com.lunarg.vulkan.debug',
])
os.environ['VULKAN_SDK'] = str(prefix)
os.environ['VK_SDK_PATH'] = str(prefix)
path_prepend('PATH', prefix / "Bin")
else:
return


@banner('Build Taichi Wheel')
def build_wheel(python: Command, pip: Command, env: dict) -> None:
def build_wheel(python: Command, pip: Command) -> None:
'''
Build the Taichi wheel
'''
pip.install('-r', 'requirements_dev.txt')
git.fetch('origin', 'master', '--tags')
proj = env['PROJECT_NAME']
proj = os.environ.get('PROJECT_NAME', 'taichi')
proj_tags = []
extra = []

if proj == 'taichi-nightly':
proj_tags.extend(['egg_info', '--tag-date'])
# Include C-API in nightly builds
env['TAICHI_CMAKE_ARGS'] += ' -DTI_WITH_C_API=ON'
os.environ['TAICHI_CMAKE_ARGS'] += ' -DTI_WITH_C_API=ON'

if platform.system() == 'Linux':
if is_manylinux2014():
Expand All @@ -95,25 +174,27 @@ def build_wheel(python: Command, pip: Command, env: dict) -> None:
python('misc/make_changelog.py', '--ver', 'origin/master', '--repo_dir',
'./', '--save')

with environ(env):
python('setup.py', *proj_tags, 'bdist_wheel', *extra)
python('setup.py', *proj_tags, 'bdist_wheel', *extra)


def main() -> None:
env = {
'PATH': os.environ['PATH'],
'LD_LIBRARY_PATH': os.environ.get('LD_LIBRARY_PATH', ''),
'TAICHI_CMAKE_ARGS': os.environ.get('TAICHI_CMAKE_ARGS', ''),
'PROJECT_NAME': os.environ.get('PROJECT_NAME', 'taichi'),
}
setup_llvm(env)
setup_vulkan(env)
sccache = setup_sccache(env)
u = platform.uname()
if (u.system, u.machine) == ('Windows', 'AMD64'):
# Use MSVC on Windows
setup_clang(as_compiler=False)
setup_msvc()
else:
# Use Clang on all other platforms
setup_clang()

setup_llvm()
setup_vulkan()
sccache = setup_sccache()

# NOTE: We use conda/venv to build wheels, which may not be the same python
# running this script.
python, pip = setup_python(env, os.environ['PY'])
build_wheel(python, pip, env)
python, pip = setup_python(os.environ['PY'])
build_wheel(python, pip)

sccache('-s')

Expand Down
91 changes: 72 additions & 19 deletions .github/workflows/scripts/ci_common/bootstrap.py
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,11 @@
# -*- coding: utf-8 -*-

# -- stdlib --
import importlib
import os
import platform
import subprocess
import sys
from pathlib import Path

# -- third party --
# -- own --


# -- code --
def is_in_venv() -> bool:
'''
Are we in a virtual environment?
Expand All @@ -19,15 +14,55 @@ def is_in_venv() -> bool:
and sys.base_prefix != sys.prefix)


def get_cache_home() -> Path:
'''
Get the cache home directory. All intermediate files should be stored here.
'''
if platform.system() == 'Windows':
return Path(os.environ['LOCALAPPDATA']) / 'build-cache'
else:
return Path.home() / '.cache' / 'build-cache'


def run(*args, env=None):
args = list(map(str, args))
if env is None:
return subprocess.Popen(args).wait()
else:
e = os.environ.copy()
e.update(env)
return subprocess.Popen(args, env=e).wait()


def restart():
'''
Restart the current process.
'''
if platform.system() == 'Windows':
# GitHub Actions will treat the step as completed when doing os.execl in Windows,
# since Windows does not have real execve, its behavior is emulated by spawning a new process and
# terminating the current process. So we do not use os.execl in Windows.
os._exit(run(sys.executable, *sys.argv))
else:
os.execl(sys.executable, sys.executable, *sys.argv)


def ensure_dependencies(fn='requirements.txt'):
'''
Automatically install dependencies if they are not installed.
'''

if 'site' in sys.modules:
sys.argv.insert(0, '-S')
restart()

p = Path(__file__).parent.parent / fn
if not p.exists():
raise RuntimeError(f'Cannot find {p}')

user = '' if is_in_venv() else '--user'
bootstrap_root = get_cache_home() / 'bootstrap'
bootstrap_root.mkdir(parents=True, exist_ok=True)
sys.path.insert(0, str(bootstrap_root))

with open(p) as f:
deps = [i.strip().split('=')[0] for i in f.read().splitlines()]
Expand All @@ -36,12 +71,17 @@ def ensure_dependencies(fn='requirements.txt'):
for dep in deps:
importlib.import_module(dep)
except ModuleNotFoundError:
print('Installing dependencies...')
if os.system(f'{sys.executable} -m pip install {user} -U pip'):
print('Installing dependencies...', flush=True)
pipcmd = [
sys.executable, '-m', 'pip', 'install',
f'--target={bootstrap_root}', '-U'
]
if run(*pipcmd, 'pip', 'setuptools'):
raise Exception('Unable to upgrade pip!')
if os.system(f'{sys.executable} -m pip install {user} -U -r {p}'):
if run(*pipcmd, '-r', p, env={'PYTHONPATH': str(bootstrap_root)}):
raise Exception('Unable to install dependencies!')
os.execl(sys.executable, sys.executable, *sys.argv)

restart()


def chdir_to_root():
Expand All @@ -51,7 +91,7 @@ def chdir_to_root():
root = Path('/')
p = Path(__file__).resolve()
while p != root:
if (p / '.git').exists():
if (p / 'setup.py').exists():
os.chdir(p)
break
p = p.parent
Expand All @@ -69,20 +109,33 @@ def set_common_env():

class _EnvironWrapper(_Environ):
def __setitem__(self, name: str, value: str) -> None:
orig = self.get(name, '')
orig = self.get(name, None)
_Environ.__setitem__(self, name, value)
new = self[name]

if orig == new:
return

from .escapes import escape_codes

G = escape_codes['bold_green']
R = escape_codes['bold_red']
N = escape_codes['reset']
print(f'{R}:: ENV -{name}={orig}{N}', file=sys.stderr, flush=True)
print(f'{G}:: ENV +{name}={new}{N}', file=sys.stderr, flush=True)

if orig == new:
pass
elif orig == None:
print(f'{G}:: ENV+ {name}={new}{N}', file=sys.stderr, flush=True)
elif new.startswith(orig):
l = len(orig)
print(f'{G}:: ENV{N} {name}={new[:l]}{G}{new[l:]}{N}',
file=sys.stderr,
flush=True)
elif new.endswith(orig):
l = len(new) - len(orig)
print(f'{G}:: ENV{N} {name}={G}{new[:l]}{N}{new[l:]}',
file=sys.stderr,
flush=True)
else:
print(f'{R}:: ENV- {name}={orig}{N}', file=sys.stderr, flush=True)
print(f'{G}:: ENV+ {name}={new}{N}', file=sys.stderr, flush=True)


def monkey_patch_environ():
Expand Down
Loading

0 comments on commit 437fe47

Please sign in to comment.