Skip to content

Commit

Permalink
Merge pull request #20 from N5N3/N5N3-patch-2
Browse files Browse the repository at this point in the history
N5N3 patch 2
  • Loading branch information
N5N3 authored Sep 29, 2021
2 parents e925450 + 480b4c5 commit 744853e
Show file tree
Hide file tree
Showing 72 changed files with 814 additions and 538 deletions.
4 changes: 1 addition & 3 deletions .buildkite/pipelines/experimental/0_webui.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
agents:
queue: "julia"
sandbox.jl: "true"

steps:
- label: ":unlock: Unlock secrets, launch pipelines"
plugins:
Expand All @@ -16,9 +15,8 @@ steps:
# pipelines.
unsigned_pipelines:
- .buildkite/pipelines/experimental/launch_unsigned_builders.yml

# Our signed pipelines must have a `signature` or `signature_file` parameter that
# verifies the treehash of the pipeline itself and the inputs listed in `inputs`
# signed_pipelines:
# - pipeline: .buildkite/pipelines/experimental/misc/foo_bar_baz.yml
# signature: "my_signature"
# signature_file: .buildkite/pipelines/experimental/misc/foo_bar_baz.yml.signature
10 changes: 3 additions & 7 deletions .buildkite/pipelines/experimental/misc/sanitizers.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,9 @@ agents:
# Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
sandbox.jl: "true"
os: "linux"

steps:
- label: "asan"
key: asan
key: "asan"
plugins:
- JuliaCI/julia#v1:
version: 1.6
Expand All @@ -17,13 +16,10 @@ steps:
gid: 1000
workspaces:
- "/cache/repos:/cache/repos"
# `contrib/check-asan.jl` needs a `julia` binary:
# `contrib/check-asan.jl` needs a `julia` binary inside the inner sandbox:
- JuliaCI/julia#v1:
version: 1.6
timeout_in_minutes: 120
# notify:
# - github_commit_status:
# context: "asan"
commands: |
echo "--- Build julia-debug with ASAN"
contrib/asan/build.sh ./tmp/test-asan -j$${JULIA_NUM_CORES} debug
contrib/asan/build.sh ./tmp/test-asan -j$${JULIA_CPU_THREADS:?} debug
1 change: 0 additions & 1 deletion .buildkite/pipelines/main/0_webui.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
agents:
queue: "julia"
sandbox.jl: "true"

steps:
- label: ":unlock: Unlock secrets, launch pipelines"
plugins:
Expand Down
9 changes: 5 additions & 4 deletions .buildkite/pipelines/main/launch_unsigned_builders.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,16 @@
steps:
- label: ":buildkite: Launch unsigned builders"
commands: |
# First, we launch the `whitespace` builder, because we want that builder to finish as quickly as possible.
# Launch the `whitespace` builder first, because we want that builder to finish as quickly as possible.
buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml
# Next, we launch the miscellaneous builders in alphabetical order.
# Launch the miscellaneous builders in alphabetical order.
buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml
buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml
buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml
# Finally, we launch the platform builders (`package_*`) and (`tester_*`) in alphabetical order.
buildkite-agent pipeline upload .buildkite/pipelines/main/platforms/linux64.yml
# Launch all of the platform builders.
bash .buildkite/pipelines/main/platforms/platforms.sh package_linux
bash .buildkite/pipelines/main/platforms/platforms.sh tester_linux
agents:
queue: julia
4 changes: 0 additions & 4 deletions .buildkite/pipelines/main/misc/doctest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ agents:
# Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
sandbox.jl: "true"
os: "linux"

steps:
- label: "doctest"
key: doctest
Expand Down Expand Up @@ -31,6 +30,3 @@ steps:
echo "--- Run Julia doctests"
JULIA_NUM_THREADS=1 make -C doc doctest=true
timeout_in_minutes: 45
notify:
- github_commit_status:
context: "doctest"
15 changes: 5 additions & 10 deletions .buildkite/pipelines/main/misc/embedding.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ agents:
# Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
sandbox.jl: "true"
os: "linux"

steps:
- label: "embedding"
key: "embedding"
Expand All @@ -20,15 +19,11 @@ steps:
- "/cache/repos:/cache/repos"
commands: |
prefix="/tmp/prefix"
echo "+++ Build julia, deploy to $${prefix}"
make -j$${JULIA_NUM_CORES} JULIA_PRECOMPILE=0 prefix=$${prefix} install
echo "+++ Build julia, deploy to $${prefix:?}"
make -j$${JULIA_CPU_THREADS:?} JULIA_PRECOMPILE=0 prefix=$${prefix:?} install
embedding_output="/tmp/embedding-test"
echo "+++ Run embedding tests, deploy to $${embedding_output}"
mkdir -p "$${embedding_output}"
make -j$${JULIA_NUM_CORES} -C test/embedding JULIA="$${prefix}/bin/julia" BIN="$${embedding_output}"
echo "+++ Run embedding tests, deploy to $${embedding_output:?}"
mkdir -p "$${embedding_output:?}"
make -j$${JULIA_CPU_THREADS:?} -C test/embedding JULIA="$${prefix:?}/bin/julia" BIN="$${embedding_output:?}"
timeout_in_minutes: 60
notify:
- github_commit_status:
context: "embedding"
20 changes: 6 additions & 14 deletions .buildkite/pipelines/main/misc/llvmpasses.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ agents:
# Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
sandbox.jl: "true"
os: "linux"

steps:
- label: "analyzegc"
key: "analyzegc"
Expand All @@ -18,15 +17,11 @@ steps:
- "/cache/repos:/cache/repos"
commands: |
echo "--- Install in-tree LLVM dependencies"
make -j$${JULIA_NUM_CORES} -C deps install-llvm install-clang install-llvm-tools install-libuv install-utf8proc install-unwind
make -j$${JULIA_CPU_THREADS:?} -C deps install-llvm install-clang install-llvm-tools install-libuv install-utf8proc install-unwind
echo "+++ run clangsa/analyzegc"
make -j$${JULIA_NUM_CORES} -C test/clangsa
make -j$${JULIA_NUM_CORES} -C src analyzegc
make -j$${JULIA_CPU_THREADS:?} -C test/clangsa
make -j$${JULIA_CPU_THREADS:?} -C src analyzegc
timeout_in_minutes: 60
notify:
- github_commit_status:
context: "analyzegc"

- label: "llvmpasses"
key: "llvmpasses"
plugins:
Expand All @@ -41,12 +36,9 @@ steps:
- "/cache/repos:/cache/repos"
commands: |
echo "--- make release"
make -j$${JULIA_NUM_CORES} release JULIA_PRECOMPILE=0
make -j$${JULIA_CPU_THREADS:?} release JULIA_PRECOMPILE=0
echo "--- make src/install-analysis-deps"
make -j$${JULIA_NUM_CORES} -C src install-analysis-deps
make -j$${JULIA_CPU_THREADS:?} -C src install-analysis-deps
echo "+++ make test/llvmpasses"
make -j$${JULIA_NUM_CORES} -C test/llvmpasses
make -j$${JULIA_CPU_THREADS:?} -C test/llvmpasses
timeout_in_minutes: 60
notify:
- github_commit_status:
context: "llvmpasses"
5 changes: 2 additions & 3 deletions .buildkite/pipelines/main/misc/whitespace.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ agents:
# Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
sandbox.jl: "true"
os: "linux"

steps:
- label: "whitespace"
key: "whitespace"
Expand All @@ -15,9 +14,9 @@ steps:
rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e"
workspaces:
- "/cache/repos:/cache/repos"
commands: |
make -j$${JULIA_NUM_CORES} check-whitespace
timeout_in_minutes: 10
notify:
- github_commit_status:
context: "whitespace"
commands: |
make -j$${JULIA_CPU_THREADS:?} check-whitespace
95 changes: 0 additions & 95 deletions .buildkite/pipelines/main/platforms/linux64.yml

This file was deleted.

6 changes: 6 additions & 0 deletions .buildkite/pipelines/main/platforms/package_linux.arches
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# ARCH ARCH_LABEL ROOTFS_ARCH TIMEOUT ROOTFS_TAG ROOTFS_TREE
# aarch64 aarch64 aarch64 60 v3.2 0566841e29f0f9880541c26a6595fd5ce0beb5ff
# armv7l armv7l armv7l 60 v3.2 fb359370b052a47ce5c84cc6b4a7a03ed7053b25
32 32 i686 60 v3.2 209c4db679a515befd7fb50ecc6bfbecf7ec3d32
# ppc64le ppc64le powerpc64le 60 v3.2 c03a0158b19d48ac84b426834fce0d3584cdd0c7
64 64 x86_64 60 v3.2 474bf61a926b2d7fcf202284d59d4b11a04601d7
49 changes: 49 additions & 0 deletions .buildkite/pipelines/main/platforms/package_linux.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
agents:
queue: "julia"
# Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
sandbox.jl: "true"
os: "linux"
steps:
- label: "package_linux${ARCH_LABEL?}"
key: package_linux${ARCH_LABEL?}
plugins:
- JuliaCI/julia#v1:
# Drop default "registries" directory, so it is not persisted from execution to execution
persist_depot_dirs: packages,artifacts,compiled
version: 1.6
- staticfloat/sandbox#v1:
rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_linux.${ROOTFS_ARCH?}.tar.gz
rootfs_treehash: "${ROOTFS_TREE?}"
uid: 1000
gid: 1000
workspaces:
# Include `/cache/repos` so that our `git` version introspection works.
- "/cache/repos:/cache/repos"
timeout_in_minutes: ${TIMEOUT?}
commands: |
echo "--- Print the full and short commit hashes"
SHORT_COMMIT_LENGTH=10
SHORT_COMMIT=`echo $${BUILDKITE_COMMIT:?} | cut -c1-$${SHORT_COMMIT_LENGTH:?}`
JULIA_BINARYDIST_FILENAME=`make print-JULIA_BINARYDIST_FILENAME | cut -c27-`
ARTIFACT_FILE_EXTENSION="tar.gz"
ARTIFACT_FILENAME="$${JULIA_BINARYDIST_FILENAME:?}.$${ARTIFACT_FILE_EXTENSION:?}"
echo "The full commit is: $${BUILDKITE_COMMIT:?}"
echo "The short commit is: $${SHORT_COMMIT:?}"
echo "The artifact filename will be: $${ARTIFACT_FILENAME:?}"
echo "--- Build Julia from source"
rm -rf $${ARTIFACT_FILENAME:?}
make -j 8
echo "--- Make sure that the working directory is clean"
if [ -z "$(git status --short)" ]; then echo "INFO: The working directory is clean."; else echo "ERROR: The working directory is dirty."; echo "Output of git status:"; git status; exit 1; fi
echo "--- Print Julia version info"
./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
echo "--- Create build artifacts"
make -j 8 binary-dist
ls -l $${ARTIFACT_FILENAME:?}
echo "--- Upload build artifacts"
buildkite-agent artifact upload $${ARTIFACT_FILENAME:?}
24 changes: 24 additions & 0 deletions .buildkite/pipelines/main/platforms/platforms.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
#!/bin/bash

SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"

PLATFORM="$1"

cat "$SCRIPT_DIR/$PLATFORM.arches" | tr -s ' ' | while read _line; do
# Remove whitespace from the beginning and end of each line
line=`echo $_line | tr -s ' '`

# Skip all lines that begin with `#`
if [[ $line == \#* ]]; then
continue
fi

export ARCH=`echo $line | cut -d ' ' -f 1`
export ARCH_LABEL=`echo $line | cut -d ' ' -f 2`
export ROOTFS_ARCH=`echo $line | cut -d ' ' -f 3`
export TIMEOUT=`echo $line | cut -d ' ' -f 4`
export ROOTFS_TAG=`echo $line | cut -d ' ' -f 5`
export ROOTFS_TREE=`echo $line | cut -d ' ' -f 6`
echo "Launching: $PLATFORM $ARCH $ARCH_LABEL $ROOTFS_ARCH $TIMEOUT"
buildkite-agent pipeline upload "$SCRIPT_DIR/$PLATFORM.yml"
done
8 changes: 8 additions & 0 deletions .buildkite/pipelines/main/platforms/tester_linux.arches
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
# ARCH ARCH_LABEL ROOTFS_ARCH TIMEOUT ROOTFS_TAG ROOTFS_TREE
# aarch64 aarch64 aarch64 60 v3.2 0566841e29f0f9880541c26a6595fd5ce0beb5ff
# armv7l armv7l armv7l 60 v3.2 fb359370b052a47ce5c84cc6b4a7a03ed7053b25
32 32 i686 60 v3.2 209c4db679a515befd7fb50ecc6bfbecf7ec3d32
# ppc64le ppc64le powerpc64le 60 v3.2 c03a0158b19d48ac84b426834fce0d3584cdd0c7
64 64_rr x86_64 180 v3.2 474bf61a926b2d7fcf202284d59d4b11a04601d7
64 64_st x86_64 60 v3.2 474bf61a926b2d7fcf202284d59d4b11a04601d7
64 64_mt x86_64 60 v3.2 474bf61a926b2d7fcf202284d59d4b11a04601d7
Loading

0 comments on commit 744853e

Please sign in to comment.