Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[ci] Switch Windows build script to build.py #6993

Merged
merged 52 commits into from
Jan 17, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
52 commits
Select commit Hold shift + click to select a range
af13a26
Temp remove testing(save time)
feisuzhu Dec 28, 2022
df4ef21
switch windows build script
feisuzhu Dec 28, 2022
5c268e5
should fail
feisuzhu Dec 28, 2022
34be3d0
setup clang
feisuzhu Dec 28, 2022
7148035
check local cache size
feisuzhu Dec 28, 2022
47804c9
tinysh use subprocess
feisuzhu Dec 28, 2022
d32838b
win miniforge fix
feisuzhu Dec 28, 2022
e78abfd
meh
feisuzhu Dec 28, 2022
d36fe31
meh
feisuzhu Dec 28, 2022
567623f
meh
feisuzhu Dec 28, 2022
711f924
conda
feisuzhu Dec 29, 2022
a52ff8d
path quirk
feisuzhu Dec 29, 2022
d09d972
win llvm strip
feisuzhu Dec 29, 2022
642f678
Revert "Temp remove testing(save time)"
feisuzhu Jan 4, 2023
7156f6d
Merge remote-tracking branch 'upstream/master' into feat/windows-build
feisuzhu Jan 4, 2023
a80dc3a
Disable tests (2nd)
feisuzhu Jan 4, 2023
1b0c151
vulkan
feisuzhu Jan 5, 2023
58ddbd3
minoor
feisuzhu Jan 6, 2023
a0a6cbd
Merge remote-tracking branch 'upstream/master' into feat/windows-build
feisuzhu Jan 6, 2023
eee4a1e
revert to plain os.environ
feisuzhu Jan 9, 2023
3f7a798
meh
feisuzhu Jan 9, 2023
54b3bce
meh
feisuzhu Jan 9, 2023
9deac6f
meh
feisuzhu Jan 9, 2023
e530911
setup msvc
feisuzhu Jan 10, 2023
f2ab6ff
Merge remote-tracking branch 'upstream/master' into feat/windows-build
feisuzhu Jan 10, 2023
e635298
auto install vc buildtools 2022
feisuzhu Jan 10, 2023
c046968
Revert "Disable tests (2nd)"
feisuzhu Jan 16, 2023
fef6e6b
Merge remote-tracking branch 'upstream/master' into feat/windows-build
feisuzhu Jan 16, 2023
951629b
meh
feisuzhu Jan 16, 2023
d9ff019
REMOVE OTHER STEPS FOR DEBUG
feisuzhu Jan 16, 2023
42b310e
use cmd
feisuzhu Jan 16, 2023
509ce05
fixup
feisuzhu Jan 16, 2023
191f28d
meh
feisuzhu Jan 16, 2023
b727a10
meh
feisuzhu Jan 16, 2023
b02ac5a
meh
feisuzhu Jan 16, 2023
3cd21bb
isolated dependency dir for build.py
feisuzhu Jan 16, 2023
120b751
clean build env
feisuzhu Jan 16, 2023
8c5fcc9
cmake env
feisuzhu Jan 16, 2023
1f555c3
more isolated
feisuzhu Jan 16, 2023
80b4b39
meh
feisuzhu Jan 16, 2023
53d95db
meh
feisuzhu Jan 17, 2023
6f11675
meh
feisuzhu Jan 17, 2023
04915b8
clang
feisuzhu Jan 17, 2023
1a62b1b
fix
feisuzhu Jan 17, 2023
a6861e4
Revert "REMOVE OTHER STEPS FOR DEBUG"
feisuzhu Jan 17, 2023
9ef900e
Merge remote-tracking branch 'upstream/master' into feat/windows-build
feisuzhu Jan 17, 2023
27956a5
remove todo
feisuzhu Jan 17, 2023
bf80b99
vs2022 packages
feisuzhu Jan 17, 2023
7f41f3f
move vulkan on to testing.yml
feisuzhu Jan 17, 2023
6896856
clean up debug messes
feisuzhu Jan 17, 2023
7e30bd2
resolve clang++ path pollution
feisuzhu Jan 17, 2023
966b26d
path escaping
feisuzhu Jan 17, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
revert to plain os.environ
  • Loading branch information
feisuzhu committed Jan 9, 2023
commit eee4a1eccc445f860331063924feaa9e7795b063
64 changes: 29 additions & 35 deletions .github/workflows/scripts/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,14 @@
from ci_common.dep import download_dep
from ci_common.misc import (banner, concat_paths, get_cache_home,
is_manylinux2014)
from ci_common.python import setup_python
from ci_common.python import path_prepend, setup_python
from ci_common.sccache import setup_sccache
from ci_common.tinysh import Command, environ, git, sh


# -- code --
@banner('Setup Clang')
def setup_clang(env_out: dict) -> None:
def setup_clang() -> None:
'''
Setup Clang.
'''
Expand All @@ -37,18 +37,18 @@ def setup_clang(env_out: dict) -> None:
out = get_cache_home() / 'clang-15'
url = 'https://github.com/python3kgae/taichi_assets/releases/download/llvm15_vs2022_clang/clang-15.0.0-win.zip'
feisuzhu marked this conversation as resolved.
Show resolved Hide resolved
download_dep(url, out)
env_out['PATH'] = concat_paths(out / 'bin', env_out.get('PATH'))
env_out[
os.environ['PATH'] = concat_paths(out / 'bin', os.environ.get('PATH'))
os.environ[
'TAICHI_CMAKE_ARGS'] += ' -DCLANG_EXECUTABLE=clang++.exe' # TODO: Can this be omitted?
env_out[
os.environ[
'TAICHI_CMAKE_ARGS'] += ' -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_C_COMPILER=clang'
bobcao3 marked this conversation as resolved.
Show resolved Hide resolved
else:
# TODO: unify all
pass


@banner('Setup LLVM')
def setup_llvm(env_out: dict) -> None:
def setup_llvm() -> None:
'''
Download and install LLVM.
'''
Expand All @@ -57,7 +57,7 @@ def setup_llvm(env_out: dict) -> None:
if 'AMDGPU_TEST' in os.environ:
# FIXME: AMDGPU bots are currently maintained separately,
# we should unify them with the rest of the bots.
env_out['LLVM_DIR'] = '/taichi-llvm-15'
os.environ['LLVM_DIR'] = '/taichi-llvm-15'
return
elif is_manylinux2014():
# FIXME: prebuilt llvm15 on ubuntu didn't work on manylinux2014 image of centos. Once that's fixed, remove this hack.
Expand All @@ -79,28 +79,28 @@ def setup_llvm(env_out: dict) -> None:
out = get_cache_home() / 'llvm15'
url = 'https://github.com/python3kgae/taichi_assets/releases/download/llvm15_vs2019_clang/taichi-llvm-15.0.0-msvc2019.zip'
feisuzhu marked this conversation as resolved.
Show resolved Hide resolved
# Could be unnecessary, commenting out for now.
# env_out['TAICHI_CMAKE_ARGS'] += " -DLLVM_AS_EXECUTABLE=llvm-as.exe"
# os.environ['TAICHI_CMAKE_ARGS'] += " -DLLVM_AS_EXECUTABLE=llvm-as.exe"
download_dep(url, out, strip=0)
else:
raise RuntimeError(f'Unsupported platform: {u.system} {u.machine}')

env_out['PATH'] = concat_paths(out / 'bin', env_out.get('PATH'))
env_out['LLVM_DIR'] = str(out)
path_prepend('PATH', out / 'bin')
os.environ['LLVM_DIR'] = str(out)


@banner('Setup Vulkan 1.3.236.0')
def setup_vulkan(env: dict):
def setup_vulkan():
u = platform.uname()
if u.system == "Linux":
url = 'https://sdk.lunarg.com/sdk/download/1.3.236.0/linux/vulkansdk-linux-x86_64-1.3.236.0.tar.gz'
prefix = get_cache_home() / 'vulkan-1.3.236.0'
download_dep(url, prefix, strip=1)
sdk = prefix / 'x86_64'
env['VULKAN_SDK'] = str(sdk)
env['PATH'] = str(sdk / "bin") + ':' + env["PATH"]
env['LD_LIBRARY_PATH'] = str(sdk / "lib") + ':' + env.get(
"LD_LIBRARY_PATH", "")
env['VK_LAYER_PATH'] = str(sdk / 'etc' / 'vulkan' / 'explicit_layer.d')
os.environ['VULKAN_SDK'] = str(sdk)
path_prepend('PATH', sdk / "bin")
path_prepend('LD_LIBRARY_PATH', sdk / 'lib')
os.environ['VK_LAYER_PATH'] = str(sdk / 'etc' / 'vulkan' /
'explicit_layer.d')
# elif (u.system, u.machine) == ("Darwin", "arm64"):
# elif (u.system, u.machine) == ("Darwin", "x86_64"):
elif (u.system, u.machine) == ('Windows', 'AMD64'):
Expand All @@ -119,28 +119,28 @@ def setup_vulkan(env: dict):
'com.lunarg.vulkan.vma',
'com.lunarg.vulkan.debug',
])
env['VULKAN_SDK'] = str(prefix)
env['VK_SDK_PATH'] = str(prefix)
env['PATH'] = str(prefix / "Bin") + ';' + env["PATH"]
os.environ['VULKAN_SDK'] = str(prefix)
os.environ['VK_SDK_PATH'] = str(prefix)
path_prepend('PATH', prefix / "Bin")
else:
return


@banner('Build Taichi Wheel')
def build_wheel(python: Command, pip: Command, env: dict) -> None:
def build_wheel(python: Command, pip: Command) -> None:
'''
Build the Taichi wheel
'''
pip.install('-r', 'requirements_dev.txt')
git.fetch('origin', 'master', '--tags')
proj = env['PROJECT_NAME']
proj = os.environ['PROJECT_NAME']
proj_tags = []
extra = []

if proj == 'taichi-nightly':
proj_tags.extend(['egg_info', '--tag-date'])
# Include C-API in nightly builds
env['TAICHI_CMAKE_ARGS'] += ' -DTI_WITH_C_API=ON'
os.environ['TAICHI_CMAKE_ARGS'] += ' -DTI_WITH_C_API=ON'

if platform.system() == 'Linux':
if is_manylinux2014():
Expand All @@ -151,26 +151,20 @@ def build_wheel(python: Command, pip: Command, env: dict) -> None:
python('misc/make_changelog.py', '--ver', 'origin/master', '--repo_dir',
'./', '--save')

with environ(env):
with environ(os.environ):
python('setup.py', *proj_tags, 'bdist_wheel', *extra)


def main() -> None:
env = {
'PATH': os.environ['PATH'],
'LD_LIBRARY_PATH': os.environ.get('LD_LIBRARY_PATH', ''),
'TAICHI_CMAKE_ARGS': os.environ.get('TAICHI_CMAKE_ARGS', ''),
'PROJECT_NAME': os.environ.get('PROJECT_NAME', 'taichi'),
}
setup_clang(env)
setup_llvm(env)
setup_vulkan(env)
sccache = setup_sccache(env)
setup_clang()
setup_llvm()
setup_vulkan()
sccache = setup_sccache()

# NOTE: We use conda/venv to build wheels, which may not be the same python
# running this script.
python, pip = setup_python(env, os.environ['PY'])
build_wheel(python, pip, env)
python, pip = setup_python(os.environ['PY'])
build_wheel(python, pip)

sccache('-s')

Expand Down
29 changes: 22 additions & 7 deletions .github/workflows/scripts/ci_common/bootstrap.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import os
import sys
from pathlib import Path
from typing import Any

# -- third party --
# -- own --
Expand Down Expand Up @@ -37,7 +38,8 @@ def ensure_dependencies(fn='requirements.txt'):
importlib.import_module(dep)
except ModuleNotFoundError:
print('Installing dependencies...')
if os.system(f'{sys.executable} -m pip install {user} -U pip'):
if os.system(
f'{sys.executable} -m pip install {user} -U pip setuptools'):
raise Exception('Unable to upgrade pip!')
if os.system(f'{sys.executable} -m pip install {user} -U -r {p}'):
raise Exception('Unable to install dependencies!')
Expand Down Expand Up @@ -69,20 +71,33 @@ def set_common_env():

class _EnvironWrapper(_Environ):
def __setitem__(self, name: str, value: str) -> None:
orig = self.get(name, '')
orig = self.get(name, None)
_Environ.__setitem__(self, name, value)
new = self[name]

if orig == new:
return

from .escapes import escape_codes

G = escape_codes['bold_green']
R = escape_codes['bold_red']
N = escape_codes['reset']
print(f'{R}:: ENV -{name}={orig}{N}', file=sys.stderr, flush=True)
print(f'{G}:: ENV +{name}={new}{N}', file=sys.stderr, flush=True)

if orig == new:
pass
elif orig == None:
print(f'{G}:: ENV+ {name}={new}{N}', file=sys.stderr, flush=True)
elif new.startswith(orig):
l = len(orig)
print(f'{G}:: ENV{N} {name}={new[:l]}{G}{new[l:]}{N}',
file=sys.stderr,
flush=True)
elif new.endswith(orig):
l = len(new) - len(orig)
print(f'{G}:: ENV{N} {name}={G}{new[:l]}{N}{new[l:]}',
file=sys.stderr,
flush=True)
else:
print(f'{R}:: ENV- {name}={orig}{N}', file=sys.stderr, flush=True)
print(f'{G}:: ENV+ {name}={new}{N}', file=sys.stderr, flush=True)


def monkey_patch_environ():
Expand Down
10 changes: 7 additions & 3 deletions .github/workflows/scripts/ci_common/misc.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,12 @@ def wrapper(*args, **kwargs):
return decorate


def concat_paths(*paths: Any) -> str:
def path_prepend(var: str, *paths: Any) -> None:
'''
Concatenate paths together.
Prepend paths to the environment variable.
'''
return os.pathsep.join(str(p) for p in paths if p)
value = os.pathsep.join(str(p) for p in paths if p)
orig = os.environ.get(var, '')
if orig:
value += os.pathsep + orig
os.environ[var] = value
16 changes: 5 additions & 11 deletions .github/workflows/scripts/ci_common/python.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
# -*- coding: utf-8 -*-

import os
import platform
import shutil
from typing import Optional, Tuple

from .dep import download_dep
from .misc import banner, concat_paths, get_cache_home
from .misc import banner, get_cache_home, path_prepend
from .tinysh import Command, sh


Expand Down Expand Up @@ -41,8 +42,7 @@ def setup_miniforge3(prefix):


@banner('Setup Python {version}')
def setup_python(env_out: dict,
version: Optional[str] = None) -> Tuple[Command, Command]:
def setup_python(version: Optional[str] = None) -> Tuple[Command, Command]:
'''
Find the required Python environment and return the `python` and `pip` commands.
'''
Expand All @@ -69,16 +69,10 @@ def setup_python(env_out: dict,
env = prefix / 'envs' / version
if windows:
exe = env / 'python.exe'
env_out['PATH'] = concat_paths(env, env / 'Scripts',
prefix / 'Library' / 'bin',
env_out.get('PATH'))
import os
os.environ['PATH'] = concat_paths(env, env / 'Scripts',
prefix / 'Library' / 'bin',
os.environ.get('PATH'))
path_prepend('PATH', env, env / 'Scripts', prefix / 'Library' / 'bin')
else:
exe = env / 'bin' / 'python'
env_out['PATH'] = concat_paths(env / 'bin', env_out.get('PATH'))
path_prepend('PATH', env / 'bin')

if not exe.exists():
conda.create('-y', '-n', version, f'python={version}')
Expand Down
10 changes: 5 additions & 5 deletions .github/workflows/scripts/ci_common/sccache.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

# -- code --
@banner("Setup sccache")
def setup_sccache(env_out: dict) -> Command:
def setup_sccache() -> Command:
"""
Download and install sccache, setup compiler wrappers, and return the `sccache` command.
"""
Expand Down Expand Up @@ -58,15 +58,15 @@ def setup_sccache(env_out: dict) -> Command:

exe.chmod(0o755)

env_out["SCCACHE_LOG"] = "error"
env_out[
os.environ["SCCACHE_LOG"] = "error"
os.environ[
"TAICHI_CMAKE_ARGS"] += f" -DCMAKE_C_COMPILER_LAUNCHER={exe} -DCMAKE_CXX_COMPILER_LAUNCHER={exe}"

# <LocalCache>
cache = root / "cache"
cache.mkdir(parents=True, exist_ok=True)
env_out["SCCACHE_DIR"] = str(cache)
env_out["SCCACHE_CACHE_SIZE"] = "40G"
os.environ["SCCACHE_DIR"] = str(cache)
os.environ["SCCACHE_CACHE_SIZE"] = "40G"
# </LocalCache>

return sh.bake(str(exe))