Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[24.1] Backport https://github.com/oracle/graal/pull/9078 #757

Merged
merged 2 commits into from
Jun 27, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
606 changes: 606 additions & 0 deletions .github/workflows/base-windows.yml

Large diffs are not rendered by default.

661 changes: 661 additions & 0 deletions .github/workflows/base.yml

Large diffs are not rendered by default.

135 changes: 135 additions & 0 deletions .github/workflows/build-quarkus.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
name: Build Quarkus

on:
workflow_call:
inputs:
artifacts-suffix:
type: string
description: 'The maven repo artifact suffix to use'
default: "null"
build-from-source:
type: boolean
description: 'Build from source or use a release'
default: true
builder-image:
type: string
description: 'The builder image to use instead of a release or building from source (e.g. quay.io/quarkus/ubi-quarkus-mandrel:20.3-java11)'
default: "null"
maven-deploy-local:
type: string
description: 'Build flag controlling whether to deploy maven artifacts locally'
default: ""
target-os:
type: string
description: 'The operating system we are building for (linux or windows)'
default: "linux"
quarkus-repo:
type: string
description: 'The Quarkus repository to be used'
default: 'quarkusio/quarkus'
quarkus-version:
type: string
description: 'Quarkus version to test (branch, tag, commit, or "latest")'
# "latest" is replaced by the latest release available in maven
default: "main"
# Builder image can't be tested on Windows due to https://github.com/actions/virtual-environments/issues/1143
# builder-image:
# description: 'The builder image to use instead of a release or building from source (e.g. quay.io/quarkus/ubi-quarkus-mandrel:20.3-java11)'
# default: "null"

env:
QUARKUS_PATH: quarkus
COMMON_MAVEN_ARGS: "-e -B --settings .github/mvn-settings.xml --fail-at-end"
MANDREL_HOME: ${{ github.workspace }}/mandrelvm

jobs:
build-quarkus:
name: Quarkus build
runs-on: ubuntu-latest
strategy:
fail-fast: false
steps:
- uses: actions/checkout@v4
with:
repository: graalvm/mandrel
fetch-depth: 1
path: workflow-mandrel
- uses: actions/checkout@v4
with:
repository: ${{ inputs.quarkus-repo }}
fetch-depth: 1
ref: ${{ inputs.quarkus-version }}
path: ${{ env.QUARKUS_PATH }}
- uses: actions/cache@v4
with:
path: ~/.m2/repository
key: ${{ inputs.target-os }}-${{ inputs.quarkus-version }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: ${{ inputs.target-os }}-${{ inputs.quarkus-version }}-maven-
- name: Change quarkus.version for Quarkus 2.2 to make mandrel-integration-test not apply quarkus_main.patch
# See https://github.com/Karm/mandrel-integration-tests/pull/64
run: |
if [ "${{ inputs.quarkus-version }}" == "2.2" ]
then
cd quarkus
bash ../workflow-mandrel/.github/update_quarkus_version.sh 2.2.999
fi
# Use Java 17 to build Quarkus as that's the lowest supported JDK version currently
- uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: '17'
- name: Download GraalVM Maven Repo
if: ${{ inputs.build-from-source == true && inputs.builder-image == 'null' && inputs.maven-deploy-local != ''}}
uses: actions/download-artifact@v4
with:
name: org-graalvm-artefacts-${{ inputs.artifacts-suffix }}
path: .
- name: Download GraalVM Maven Version
if: ${{ inputs.build-from-source == true && inputs.builder-image == 'null' && inputs.maven-deploy-local != ''}}
uses: actions/download-artifact@v4
with:
name: mandrel-maven-version-${{ inputs.artifacts-suffix }}
path: .
- name: Extract GraalVM Maven Repo and GraalVM Maven Version
if: ${{ inputs.build-from-source == true && inputs.builder-image == 'null' && inputs.maven-deploy-local != ''}}
run: |
tar -xzvf graalvm-maven-artefacts.tgz -C ~
tar -xzvf graalvm-version.tgz -C $(dirname ${MANDREL_HOME})
- name: Build quarkus with local graalvm version
if: ${{ inputs.build-from-source == true && inputs.builder-image == 'null' && inputs.maven-deploy-local != ''}}
run: |
rm -f maven_graalvm_before_build.txt maven_graalvm_after_build.txt
find ~/.m2/repository/org/graalvm | sort > maven_graalvm_before_build.txt
GRAAL_MVN_ARTIFACTS_VERS=$(cat ${MANDREL_HOME}/.maven-version)
echo "Building quarkus with locally installed GraalVM maven artefacts in version: ${GRAAL_MVN_ARTIFACTS_VERS}"
cd ${QUARKUS_PATH}
./mvnw ${COMMON_MAVEN_ARGS} -Dquickly -Dgraal-sdk.version="${GRAAL_MVN_ARTIFACTS_VERS}"
cd -
find ~/.m2/repository/org/graalvm | sort > maven_graalvm_after_build.txt
diff -u maven_graalvm_before_build.txt maven_graalvm_after_build.txt
- name: Build quarkus with default graalvm version
if: ${{ inputs.build-from-source == false || inputs.builder-image != 'null' || inputs.maven-deploy-local == ''}}
run: |
cd ${QUARKUS_PATH}
./mvnw ${COMMON_MAVEN_ARGS} -Dquickly
cd -
- name: Tar Maven Repo
if: inputs.target-os != 'windows'
run: |
tar -czvf maven-repo.tgz -C ~ .m2/repository
- name: Tar Maven Repo (windows)
if: inputs.target-os == 'windows'
run: |
tar -I 'pigz -9' -cf maven-repo.tgz -C ~ .m2/repository
- name: Persist Maven Repo
uses: actions/upload-artifact@v4
with:
name: ${{ inputs.target-os }}-maven-repo-${{ inputs.artifacts-suffix }}
path: maven-repo.tgz
- name: Delete Local Quarkus Artifacts From Cache
run: |
rm -r ~/.m2/repository/io/quarkus
- name: Delete Local GraalVM Artifacts From Cache
if: ${{ inputs.build-from-source == true && inputs.builder-image == 'null' && inputs.maven-deploy-local != ''}}
run: |
rm -rf ~/.m2/repository/org/graalvm
47 changes: 47 additions & 0 deletions .github/workflows/mandrel.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
name: Mandrel-Quarkus tests

on:
push:
paths-ignore:
- '.github/workflows/main.yml'
- '.github/workflows/quarkus.yml'
- '**.md'
pull_request:
paths-ignore:
- '.github/workflows/main.yml'
- '.github/workflows/quarkus.yml'
- '**.md'
workflow_dispatch:

# The following aims to reduce CI CPU cycles by:
# 1. Cancelling any previous builds of this PR when pushing new changes to it
# 2. Cancelling any previous builds of a branch when pushing new changes to it in a fork
# 3. Cancelling any pending builds, but not active ones, when pushing to a branch in the main
# repository. This prevents us from constantly cancelling CI runs, while being able to skip
# intermediate builds. E.g., if we perform two pushes the first one will start a CI job and
# the second one will add another one to the queue; if we perform a third push while the
# first CI job is still running the previously queued CI job (for the second push) will be
# cancelled and a new CI job will be queued for the latest (third) push.
concurrency:
group: "workflow = ${{ github.workflow }}, ref = ${{ github.event.ref }}, pr = ${{ github.event.pull_request.id }}"
cancel-in-progress: ${{ github.event_name == 'pull_request' || github.repository != 'graalvm/mandrel' }}

jobs:
q-main-ea:
name: "Q main M 24.1 EA"
uses: ./.github/workflows/base.yml
with:
quarkus-version: "main"
repo: ${{ github.repository }}
version: ${{ github.ref }}
mandrel-packaging-version: "24.1"
jdk: "23/ea"
q-main-ea-win:
name: "Q main M 24.1 windows EA"
uses: ./.github/workflows/base-windows.yml
with:
quarkus-version: "main"
repo: ${{ github.repository }}
version: ${{ github.ref }}
mandrel-packaging-version: "24.1"
jdk: "23/ea"
46 changes: 46 additions & 0 deletions .github/workflows/native-tests-stats-upload.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
name: Native tests stats upload

on:
workflow_call:
inputs:
artifacts-suffix:
type: string
description: 'The stats artifact suffix to download'
default: "null"
build-stats-tag:
type: string
description: 'The tag to use for build stats upload of native tests (e.g. 22.3.0-dev-jdk17-mytest-patch-before)'
default: "null"
category:
type: string
description: 'The native integration test category to upload stats for'
default: "null"
secrets:
UPLOAD_COLLECTOR_TOKEN:
description: 'A token used to report build statistics to a collector'
required: false

jobs:
native-tests-stats-upload:
name: Upload build stats to collector
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
repository: graalvm/mandrel
fetch-depth: 1
path: workflow-mandrel
- uses: actions/download-artifact@v4
with:
name: build-stats-${{inputs.category}}-${{ inputs.artifacts-suffix }}
path: .
- name: Extract and import build stats
env:
BUILD_STATS_TAG: ${{ inputs.build-stats-tag }}
UPLOAD_TOKEN: ${{ secrets.UPLOAD_COLLECTOR_TOKEN }}
COLLECTOR_URL: https://collector.foci.life/api/v1/image-stats
shell: bash
run: |
tar -xf build-stats.tgz
echo "Tag for stat upload is going to be: '${BUILD_STATS_TAG}'"
DIR=quarkus/ TAG="${BUILD_STATS_TAG}" TOKEN="${UPLOAD_TOKEN}" URL="${COLLECTOR_URL}" bash workflow-mandrel/.github/import_stats.sh
76 changes: 76 additions & 0 deletions README-Mandrel.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
# Mandrel

Mandrel is [a downstream distribution of the GraalVM community edition](https://developers.redhat.com/blog/2020/06/05/mandrel-a-community-distribution-of-graalvm-for-the-red-hat-build-of-quarkus/).
Mandrel's main goal is to provide a `native-image` release specifically to support [Quarkus](https://quarkus.io).
The aim is to align the `native-image` capabilities from GraalVM with OpenJDK and Red Hat Enterprise Linux libraries to improve maintainability for native Quarkus applications.
Mandrel can best be described as a distribution of a regular OpenJDK with a specially packaged GraalVM Native Image builder (`native-image`).

## How Does Mandrel Differ From Graal

Mandrel releases are built from a code base derived from the upstream GraalVM code base, with only minor changes but some significant exclusions.
A full distribution of GraalVM is much more than `native-image`: it has polyglot support, the Truffle framework which allows for efficient implementation of interpreters, an LLVM compiler back end for native image, the libgraal JIT compiler as a replacement for Hotspot’s C2 server compiler and much more.
Mandrel is the small subset of that functionality we support for the `native-image` use-case.

Mandrel's `native-image` also doesn't include the following features:
* The experimental image-build server, i.e., the `--experimental-build-server` option.
* The LLVM backend, i.e., the `-H:CompilerBackend=llvm` option.
* The musl libc implementation, i.e., the `--libc=musl` option.
* Support for generating static native images, i.e., the `--static` option.
* Support for non JVM-based languages and polyglot, i.e., the `--language:<languageId>` option.

Mandrel is also built slightly differently to GraalVM, using the standard OpenJDK project release of jdk17u.
This means it does not profit from a few small enhancements that Oracle have added to the version of OpenJDK used to build their own GraalVM downloads.
Most of these enhancements are to the JVMCI module that allows the Graal compiler to be run inside OpenJDK.
The others are small cosmetic changes to behaviour.
These enhancements may in some cases cause minor differences in the progress of native image generation.
They should not cause the resulting images themselves to execute in a noticeably different manner.

## Communication Channels

* [Slack](https://www.graalvm.org/slack-invitation) - Join `#mandrel` channel at graalvm's slack workspace
* [graalvm-dev@oss.oracle.com](mailto:graalvm-dev@oss.oracle.com?subject=[MANDREL]) mailing list - Subscribe [here](https://oss.oracle.com/mailman/listinfo/graalvm-dev)
* [GitHub issues](https://github.com/graalvm/mandrel/issues) for bug reports, questions, or requests for enhancements.

Please report security vulnerabilities according to the [Reporting Vulnerabilities guide](https://www.oracle.com/corporate/security-practices/assurance/vulnerability/reporting.html).

## Getting Started

Mandrel distributions can be downloaded from [the repository's releases](https://github.com/graalvm/mandrel/releases)
and container images are available at [quay.io](https://quay.io/repository/quarkus/ubi-quarkus-mandrel-builder-image?tag=latest&tab=tags).

### Prerequisites

Mandrel's `native-image` depends on the following packages:
* freetype-devel
* gcc
* glibc-devel
* libstdc++-static
* zlib-devel

On Fedora/CentOS/RHEL they can be installed with:
```bash
dnf install glibc-devel zlib-devel gcc freetype-devel libstdc++-static
```

**Note**: The package might be called `glibc-static` or `libstdc++-devel` instead of `libstdc++-static` depending on your system.
If the system is missing stdc++, `gcc-c++` package is needed too.

On Ubuntu-like systems with:
```bash
apt install g++ zlib1g-dev libfreetype6-dev
```

## Building Mandrel From Source

For building Mandrel from source please see [mandrel-packaging](https://github.com/graalvm/mandrel-packaging)
and consult [Repository Structure in CONTRIBUTING.md](CONTRIBUTING.md#repository-structure) regarding which branch of Mandrel to use.

# Community
Empowering Software Development with Works on Arm Initiative
![Works on Arm logo](img/works_on_arm_900x77.png)
[Works on Arm](https://www.arm.com/solutions/infrastructure/works-on-arm) is a strategic initiative to enable and accelerate the
software ecosystem for Arm64.

[GraalVM's](https://www.graalvm.org/) [Mandrel](https://github.com/graalvm/mandrel) distribution that
powers [Quarkus Native](https://quarkus.io/guides/building-native-image) proudly counts itself among the libraries and
tools that successfully leveraged the resources from Works on Arm.
16 changes: 8 additions & 8 deletions common.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

"COMMENT.jdks": "When adding or removing JDKs keep in sync with JDKs in ci/common.jsonnet",
"jdks": {
"galahad-jdk": {"name": "jpg-jdk", "version": "23", "build_id": "jdk-23+25-2038", "platformspecific": true, "extrabundles": ["static-libs"]},
"galahad-jdk": {"name": "jpg-jdk", "version": "23", "build_id": "jdk-23+26-2139", "platformspecific": true, "extrabundles": ["static-libs"]},

"oraclejdk11": {"name": "jpg-jdk", "version": "11.0.11", "build_id": "jdk-11.0.11+9", "platformspecific": true, "extrabundles": ["static-libs"] },

Expand Down Expand Up @@ -45,13 +45,13 @@
"labsjdk-ee-21-llvm": {"name": "labsjdk", "version": "ee-21.0.2+13-jvmci-23.1-b33-sulong", "platformspecific": true },
"graalvm-ee-21": {"name": "graalvm-java21", "version": "23.1.3", "platformspecific": true },

"oraclejdk-latest": {"name": "jpg-jdk", "version": "23", "build_id": "jdk-23+25", "platformspecific": true, "extrabundles": ["static-libs"]},
"labsjdk-ce-latest": {"name": "labsjdk", "version": "ce-23+25-jvmci-b01", "platformspecific": true },
"labsjdk-ce-latestDebug": {"name": "labsjdk", "version": "ce-23+25-jvmci-b01-debug", "platformspecific": true },
"labsjdk-ce-latest-llvm": {"name": "labsjdk", "version": "ce-23+25-jvmci-b01-sulong", "platformspecific": true },
"labsjdk-ee-latest": {"name": "labsjdk", "version": "ee-23+25-jvmci-b01", "platformspecific": true },
"labsjdk-ee-latestDebug": {"name": "labsjdk", "version": "ee-23+25-jvmci-b01-debug", "platformspecific": true },
"labsjdk-ee-latest-llvm": {"name": "labsjdk", "version": "ee-23+25-jvmci-b01-sulong", "platformspecific": true }
"oraclejdk-latest": {"name": "jpg-jdk", "version": "23", "build_id": "jdk-23+26", "platformspecific": true, "extrabundles": ["static-libs"]},
"labsjdk-ce-latest": {"name": "labsjdk", "version": "ce-23+26-jvmci-b01", "platformspecific": true },
"labsjdk-ce-latestDebug": {"name": "labsjdk", "version": "ce-23+26-jvmci-b01-debug", "platformspecific": true },
"labsjdk-ce-latest-llvm": {"name": "labsjdk", "version": "ce-23+26-jvmci-b01-sulong", "platformspecific": true },
"labsjdk-ee-latest": {"name": "labsjdk", "version": "ee-23+26-jvmci-b01", "platformspecific": true },
"labsjdk-ee-latestDebug": {"name": "labsjdk", "version": "ee-23+26-jvmci-b01-debug", "platformspecific": true },
"labsjdk-ee-latest-llvm": {"name": "labsjdk", "version": "ee-23+26-jvmci-b01-sulong", "platformspecific": true }
},

"eclipse": {
Expand Down
2 changes: 1 addition & 1 deletion compiler/mx.compiler/suite.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
"sourceinprojectwhitelist" : [],

"groupId" : "org.graalvm.compiler",
"version" : "24.1.0",
"version" : "24.1.0.0",
"release" : False,
"url" : "http://www.graalvm.org/",
"developer" : {
Expand Down
Loading
Loading