Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use the correct Docker Image for RocksJava on Linux #12169

Closed
wants to merge 8 commits into from
21 changes: 11 additions & 10 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -146,9 +146,12 @@ commands:
install-maven:
steps:
- run:
name: Install maven
name: Install Maven
command: |
sudo apt-get update -y && sudo apt-get install -y maven
wget --no-check-certificate https://dlcdn.apache.org/maven/maven-3/3.9.6/binaries/apache-maven-3.9.6-bin.tar.gz
tar zxf apache-maven-3.9.6-bin.tar.gz
echo "export M2_HOME=$(pwd)/apache-maven-3.9.6" >> $BASH_ENV
echo 'export PATH=$M2_HOME/bin:$PATH' >> $BASH_ENV

setup-folly:
steps:
Expand Down Expand Up @@ -231,6 +234,7 @@ executors:
- image: zjay437/rocksdb:0.6
linux-java-docker:
docker:
# This is the Docker Image used for building RocksJava releases, see: https://github.com/evolvedbinary/docker-rocksjava
- image: evolvedbinary/rocksjava:centos6_x64-be

jobs:
Expand Down Expand Up @@ -623,7 +627,7 @@ jobs:
- windows-build-steps

build-linux-java:
executor: linux-docker
executor: linux-java-docker
resource_class: large
steps:
- pre-steps
Expand All @@ -636,17 +640,13 @@ jobs:
which javac && javac -version
- run:
name: "Test RocksDBJava"
command: make V=1 J=8 -j8 jtest
command: scl enable devtoolset-7 'make V=1 J=8 -j8 jtest'
- post-steps

build-linux-java-pmd:
machine:
image: ubuntu-2004:202111-02
executor: linux-java-docker
resource_class: large
environment:
JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
steps:
- install-maven
- pre-steps
- run:
name: "Set Java Environment"
Expand All @@ -655,9 +655,10 @@ jobs:
echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
which java && java -version
which javac && javac -version
- install-maven
- run:
name: "PMD RocksDBJava"
command: make V=1 J=8 -j8 jpmd
command: scl enable devtoolset-7 'make V=1 J=8 -j8 jpmd'
- post-pmd-steps

build-linux-java-static:
Expand Down
8 changes: 6 additions & 2 deletions .github/actions/install-maven/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@ name: install-maven
runs:
using: composite
steps:
- name: Install maven
run: sudo apt-get update -y && sudo apt-get install -y maven
- name: Install Maven
run: |
wget --no-check-certificate https://dlcdn.apache.org/maven/maven-3/3.9.6/binaries/apache-maven-3.9.6-bin.tar.gz
tar zxf apache-maven-3.9.6-bin.tar.gz
echo "export M2_HOME=$(pwd)/apache-maven-3.9.6" >> $GITHUB_ENV
echo "$(pwd)/apache-maven-3.9.6/bin" >> $GITHUB_PATH
shell: bash
7 changes: 5 additions & 2 deletions .github/actions/post-benchmarks/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,12 @@ name: post-benchmarks
runs:
using: composite
steps:
- uses: actions/upload-artifact@v3.1.3
- name: Upload Benchmark Results artifact
uses: actions/upload-artifact@v4.0.0
with:
path: "${{ runner.temp }}/benchmark-results"
name: benchmark-results
path: "${{ runner.temp }}/benchmark-results/**"
if-no-files-found: error
- name: Send benchmark report to visualisation
run: |-
set +e
Expand Down
10 changes: 0 additions & 10 deletions .github/actions/post-pmd-steps/action.yml

This file was deleted.

41 changes: 27 additions & 14 deletions .github/actions/post-steps/action.yml
Original file line number Diff line number Diff line change
@@ -1,25 +1,38 @@
name: post-steps
description: Steps that are taken after a RocksDB job
inputs:
artifact-prefix:
description: Prefix to append to the name of artifacts that are uploaded
required: true
default: "${{ github.job }}"
runs:
using: composite
steps:
- uses: actions/upload-artifact@v3.1.3
- name: Upload Test Results artifact
uses: actions/upload-artifact@v4.0.0
with:
path: "${{ runner.temp }}/test-results"
- uses: actions/upload-artifact@v3.1.3
name: "${{ inputs.artifact-prefix }}-test-results"
path: "${{ runner.temp }}/test-results/**"
- name: Upload DB LOG file artifact
uses: actions/upload-artifact@v4.0.0
with:
name: "${{ inputs.artifact-prefix }}-db-log-file"
path: LOG
- name: Compress Test Logs
run: tar -cvzf t.tar.gz t
- name: Copy Test Logs (on Failure)
if: ${{ failure() }}
run: |
mkdir -p ${{ runner.temp }}/failure-test-logs
cp -r t/* ${{ runner.temp }}/failure-test-logs
shell: bash
- uses: actions/upload-artifact@v3.1.3
- name: Upload Test Logs (on Failure) artifact
uses: actions/upload-artifact@v4.0.0
with:
path: t.tar.gz
- run: |-
mkdir -p ${{ runner.temp }}/core_dumps
cp core.* ${{ runner.temp }}/core_dumps
if: ${{ failure() }}
shell: bash
- uses: actions/upload-artifact@v3.1.3
name: "${{ inputs.artifact-prefix }}-failure-test-logs"
path: ${{ runner.temp }}/failure-test-logs/**
if-no-files-found: ignore
- name: Upload Core Dumps artifact
uses: actions/upload-artifact@v4.0.0
with:
path: "${{ runner.temp }}/core_dumps"
name: "${{ inputs.artifact-prefix }}-core-dumps"
path: "core.*"
if-no-files-found: ignore
38 changes: 29 additions & 9 deletions .github/workflows/pr-jobs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -324,8 +324,9 @@ jobs:
- name: compress test report
run: tar -cvzf scan_build_report.tar.gz scan_build_report
if: failure()
- uses: actions/upload-artifact@v3.1.3
- uses: actions/upload-artifact@v4.0.0
with:
name: scan-build-report
path: scan_build_report.tar.gz
build-linux-unity-and-headers:
if: ${{ github.repository_owner == 'facebook' }}
Expand Down Expand Up @@ -461,19 +462,29 @@ jobs:
runs-on:
labels: 4-core-ubuntu
container:
image: zjay437/rocksdb:0.6
image: evolvedbinary/rocksjava:centos6_x64-be
options: --shm-size=16gb
steps:
- uses: actions/checkout@v4.1.0
# The docker image is intentionally based on an OS that has an older GLIBC version.
# That GLIBC is incompatibile with GitHub's actions/checkout. Thus we implement a manual checkout step.
- name: Checkout
env:
GH_TOKEN: ${{ github.token }}
run: |
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is copy-pasting a complex piece of functionality. Presumably it can go into a sharable "uses" step.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@pdillinger I did try that, but unfortunately it is not possible as you end up in a "chicken or egg" situation. i.e. if you put the code in a separate GitHub action file of this repo, then it isn't yet available as the code hasn't been checked out from Git until after this step would run.

chown `whoami` . || true
git clone --no-checkout https://oath2:$GH_TOKEN@github.com/${{ github.repository }}.git .
git -c protocol.version=2 fetch --update-head-ok --no-tags --prune --no-recurse-submodules --depth=1 origin +${{ github.sha }}:${{ github.ref }}
git checkout --progress --force ${{ github.ref }}
git log -1 --format='%H'
- uses: "./.github/actions/pre-steps"
- name: Set Java Environment
run: |-
echo "JAVA_HOME=${JAVA_HOME}"
which java && java -version
which javac && javac -version
- name: Test RocksDBJava
run: make V=1 J=8 -j8 jtest
- uses: "./.github/actions/post-steps"
run: scl enable devtoolset-7 'make V=1 J=8 -j8 jtest'
# NOTE: post-steps skipped because of compatibility issues with docker image
build-linux-java-static:
if: ${{ github.repository_owner == 'facebook' }}
runs-on:
Expand All @@ -482,9 +493,8 @@ jobs:
image: evolvedbinary/rocksjava:centos6_x64-be
options: --shm-size=16gb
steps:
# The docker image is based on such an old OS that it has a GLIBC
# incompatibility with actions/checkout and other actions. Thus we
# implement a manual checkout step.
# The docker image is intentionally based on an OS that has an older GLIBC version.
# That GLIBC is incompatibile with GitHub's actions/checkout. Thus we implement a manual checkout step.
- name: Checkout
env:
GH_TOKEN: ${{ github.token }}
Expand Down Expand Up @@ -574,6 +584,9 @@ jobs:
if: ${{ github.repository_owner == 'facebook' }}
runs-on:
labels: 4-core-ubuntu
container:
image: evolvedbinary/rocksjava:rockylinux8_x64-be
options: --shm-size=16gb
steps:
- uses: actions/checkout@v4.1.0
- uses: "./.github/actions/install-maven"
Expand All @@ -585,4 +598,11 @@ jobs:
which javac && javac -version
- name: PMD RocksDBJava
run: make V=1 J=8 -j8 jpmd
- uses: "./.github/actions/post-pmd-steps"
- uses: actions/upload-artifact@v4.0.0
with:
name: pmd-report
path: "${{ github.workspace }}/java/target/pmd.xml"
- uses: actions/upload-artifact@v4.0.0
with:
name: maven-site
path: "${{ github.workspace }}/java/target/site"
Loading