Skip to content
CI

feat: avoid need data lock in batch queue #18855

Sign in to view logs

feat: avoid need data lock in batch queue

feat: avoid need data lock in batch queue #18855

Workflow file for this run

.github/workflows/ci.yml at 84b184d
name: CI
on:
pull_request:
paths-ignore:
- 'docs/**'
- 'README.md'
#on:
# push:
# branches-ignore:
# - '**' # temporally ignore all
jobs:
commit-lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2.5.0
with:
fetch-depth: 0
- uses: wagoid/commitlint-github-action@v4
lint-flake-8:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2.5.0
- name: Set up Python 3.8
uses: actions/setup-python@v4
with:
python-version: 3.8
- name: Lint with flake8
run: |
pip install flake8
# stop the build if there are Python syntax errors or undefined names
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --exclude .git,__pycache__,docs/source/conf.py,old,build,dist,tests/,jina/resources/,jina/proto/docarray_v1,jina/proto/docarray_v2,jina/serve/consensus/add_voter/pb,jina/serve/consensus/add_voter/pb2
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude .git,__pycache__,docs/source/conf.py,old,build,dist,tests/,jina/proto/docarray_v1,jina/proto/docarray_v2,jina/serve/consensus/add_voter/pb,jina/serve/consensus/add_voter/pb2
check-docstring:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2.5.0
with:
fetch-depth: 0
- name: Set up Python 3.8
uses: actions/setup-python@v1
with:
python-version: 3.8
- id: file_changes
uses: Ana06/get-changed-files@v1.2
- name: docstring check with darglint and pydocstyle
run: ./scripts/docstrings_lint.sh
env:
CHANGED_FILES: ${{ steps.file_changes.outputs.added_modified }}
check-black:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2.5.0
with:
fetch-depth: 0
- name: Set up Python 3.8
uses: actions/setup-python@v4
with:
python-version: 3.8
- id: file_changes
uses: Ana06/get-changed-files@v1.2
- name: check black
run: ./scripts/black.sh
env:
CHANGED_FILES: ${{ steps.file_changes.outputs.added_modified }}
code-injection:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2.5.0
if: ${{ !github.event.pull_request.head.repo.fork }}
with:
token: ${{ secrets.GITHUB_TOKEN }}
ref: ${{ github.event.pull_request.head.sha }}
- name: Set up Python 3.8
uses: actions/setup-python@v4
if: ${{ !github.event.pull_request.head.repo.fork }}
with:
python-version: 3.8
- name: Styling
id: styling
if: ${{ !github.event.pull_request.head.repo.fork }}
run: |
git config --local user.email "dev-bot@jina.ai"
git config --local user.name "Jina Dev Bot"
pip install ".[devel,black]"
pip install setuptools
cd scripts && bash ./devbot.sh && cd -
if [[ -z $(git status -s) ]]
then
echo "tree is clean"
else
git add -u
git commit -m "style: fix overload and cli autocomplete"
echo "IS_CHANGED=YES" >> $GITHUB_OUTPUT
fi
- name: Push changes to head ref
uses: ad-m/github-push-action@v0.6.0
if: ${{ !github.event.pull_request.head.repo.fork }}
with:
github_token: ${{ secrets.JINA_DEV_BOT }}
branch: ${{ github.head_ref }}
- name: Fail this pipelint
if: ${{ steps.styling.outputs.IS_CHANGED == 'YES' && !github.event.pull_request.head.repo.fork }}
run: exit 1
docker-image-test:
needs: [code-injection]
runs-on: ubuntu-latest
services:
registry:
image: registry:2
ports:
- 5000:5000
strategy:
fail-fast: false
matrix:
test-arch: ["linux/amd64", "linux/arm64"]
steps:
# - name: Cancel Previous Runs
# uses: styfle/cancel-workflow-action@0.9.0
# with:
# access_token: ${{ github.token }}
- uses: actions/checkout@v2.5.0
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v1
with:
install: true
driver-opts: network=host
- run: |
docker run --privileged --rm tonistiigi/binfmt --uninstall qemu-aarch64
docker run --rm --privileged tonistiigi/binfmt --install all
- name: Build and test
uses: docker/build-push-action@v2
with:
context: .
file: Dockerfiles/debianx.Dockerfile
platforms: ${{ matrix.test-arch }}
push: true
tags: localhost:5000/jina/multiarch:latest
target: jina
- run: |
docker run --platform ${{ matrix.test-arch }} localhost:5000/jina/multiarch:latest -v
hub-test:
runs-on: ubuntu-latest
needs: [lint-flake-8, code-injection]
if: ${{ !github.event.pull_request.head.repo.fork }}
steps:
# - name: Cancel Previous Runs
# uses: styfle/cancel-workflow-action@0.9.0
# with:
# access_token: ${{ github.token }}
- uses: actions/checkout@v2.5.0
- name: Set up Python 3.8
uses: actions/setup-python@v4
with:
python-version: 3.8
- name: Test hubapp with hubpods
run: |
./tests/jinahub/test_integration.sh
timeout-minutes: 30
env:
JINAHUB_USERNAME: ${{ secrets.JINAHUB_USERNAME }}
JINAHUB_PASSWORD: ${{ secrets.JINAHUB_PASSWORD }}
k8s-flow-test:
needs: [lint-flake-8, code-injection]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2.5.0
- name: Set up Python 3.8
uses: actions/setup-python@v4
with:
python-version: 3.8
- name: Prepare enviroment
run: |
docker build --build-arg DOCARRAY_VERSION="0.21.0" -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip .
python -m pip install --upgrade pip
python -m pip install wheel
pip install ".[all]" --no-cache-dir
pip install docarray==0.21.0
jina
export JINA_LOG_LEVEL="ERROR"
curl -LO https://dl.k8s.io/release/v1.21.1/bin/linux/amd64/kubectl
sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Test k8s Flow
run: |
export LINKERD2_VERSION=stable-2.11.4
curl --proto '=https' --tlsv1.2 -sSfL https://run.linkerd.io/install | sh
pytest -v -s --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml ./tests/k8s/test_k8s_flow.py
timeout-minutes: 30
- name: Check codecov file
id: check_files
uses: andstor/file-existence-action@v1
with:
files: "coverage.xml"
- name: Upload coverage from test to Codecov
uses: codecov/codecov-action@v3.1.1
if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8'
with:
file: coverage.xml
name: ${{ matrix.test-path }}-codecov
flags: ${{ steps.test.outputs.codecov_flag }}
fail_ci_if_error: false
k8s-deployment-test:
needs: [lint-flake-8, code-injection]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2.5.0
- name: Set up Python 3.8
uses: actions/setup-python@v4
with:
python-version: 3.8
- name: Prepare enviroment
run: |
docker build --build-arg DOCARRAY_VERSION="0.21.0" -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip .
python -m pip install --upgrade pip
python -m pip install wheel
pip install ".[all]" --no-cache-dir
pip install docarray==0.21.0
jina
export JINA_LOG_LEVEL="ERROR"
curl -LO https://dl.k8s.io/release/v1.21.1/bin/linux/amd64/kubectl
sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Test k8s Deployment
run: |
export LINKERD2_VERSION=stable-2.11.4
curl --proto '=https' --tlsv1.2 -sSfL https://run.linkerd.io/install | sh
pytest -v -s --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml ./tests/k8s/test_k8s_deployment.py
timeout-minutes: 30
- name: Check codecov file
id: check_files
uses: andstor/file-existence-action@v1
with:
files: "coverage.xml"
- name: Upload coverage from test to Codecov
uses: codecov/codecov-action@v3.1.1
if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8'
with:
file: coverage.xml
name: ${{ matrix.test-path }}-codecov
flags: ${{ steps.test.outputs.codecov_flag }}
fail_ci_if_error: false
k8s-graceful-test:
needs: [lint-flake-8, code-injection]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2.5.0
- name: Set up Python 3.8
uses: actions/setup-python@v4
with:
python-version: 3.8
- name: Prepare enviroment
run: |
docker build --build-arg DOCARRAY_VERSION="0.21.0" -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip .
python -m pip install --upgrade pip
python -m pip install wheel
pip install ".[all]" --no-cache-dir
pip install docarray==0.21.0
jina
export JINA_LOG_LEVEL="ERROR"
curl -LO https://dl.k8s.io/release/v1.21.1/bin/linux/amd64/kubectl
sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Test graceful request handling on k8s
run: |
export LINKERD2_VERSION=stable-2.11.4
curl --proto '=https' --tlsv1.2 -sSfL https://run.linkerd.io/install | sh
pytest -v -s --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml ./tests/k8s/test_k8s_graceful_request_handling.py
timeout-minutes: 45
- name: Check codecov file
id: check_files
uses: andstor/file-existence-action@v1
with:
files: "coverage.xml"
- name: Upload coverage from test to Codecov
uses: codecov/codecov-action@v3.1.1
if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8'
with:
file: coverage.xml
name: ${{ matrix.test-path }}-codecov
flags: ${{ steps.test.outputs.codecov_flag }}
fail_ci_if_error: false
k8s-failures-test:
needs: [ lint-flake-8, code-injection ]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2.5.0
- name: Set up Python 3.8
uses: actions/setup-python@v4
with:
python-version: 3.8
- name: Prepare enviroment
run: |
docker build --build-arg DOCARRAY_VERSION="0.21.0" -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip .
python -m pip install --upgrade pip
python -m pip install wheel
pip install ".[all]" --no-cache-dir
pip install docarray==0.21.0
jina
export JINA_LOG_LEVEL="ERROR"
curl -LO https://dl.k8s.io/release/v1.21.1/bin/linux/amd64/kubectl
sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Test k8s failures
run: |
export LINKERD2_VERSION=stable-2.11.4
curl --proto '=https' --tlsv1.2 -sSfL https://run.linkerd.io/install | sh
curl --proto '=https' --tlsv1.2 -sSfL https://linkerd.github.io/linkerd-smi/install | sh
pytest -v -s --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml ./tests/k8s/test_k8s_failures.py
timeout-minutes: 45
- name: Check codecov file
id: check_files
uses: andstor/file-existence-action@v1
with:
files: "coverage.xml"
- name: Upload coverage from test to Codecov
uses: codecov/codecov-action@v3.1.1
if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8'
with:
file: coverage.xml
name: ${{ matrix.test-path }}-codecov
flags: ${{ steps.test.outputs.codecov_flag }}
fail_ci_if_error: false
k8s-otel-test:
needs: [lint-flake-8, code-injection]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2.5.0
- name: Set up Python 3.8
uses: actions/setup-python@v4
with:
python-version: 3.8
- name: Prepare enviroment
run: |
docker build --build-arg DOCARRAY_VERSION="0.21.0" -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip .
python -m pip install --upgrade pip
python -m pip install wheel
pip install ".[all]" --no-cache-dir
pip install docarray==0.21.0
jina
export JINA_LOG_LEVEL="ERROR"
curl -LO https://dl.k8s.io/release/v1.21.1/bin/linux/amd64/kubectl
sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Test K8s with OpenTelemetry
run: |
pytest -v -s --cov=jina --cov-report=xml ./tests/k8s_otel
timeout-minutes: 30
- name: Check codecov file
id: check_files
uses: andstor/file-existence-action@v1
with:
files: "coverage.xml"
- name: Upload coverage from test to Codecov
uses: codecov/codecov-action@v3.1.1
if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8'
with:
file: coverage.xml
name: ${{ matrix.test-path }}-codecov
flags: ${{ steps.test.outputs.codecov_flag }}
fail_ci_if_error: false
docker-compose-test:
needs: [lint-flake-8, code-injection]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2.5.0
- name: Set up Python 3.8
uses: actions/setup-python@v4
with:
python-version: 3.8
- name: Prepare enviroment
run: |
docker build --build-arg DOCARRAY_VERSION="0.21.0" -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip .
python -m pip install --upgrade pip
python -m pip install wheel
pip install ".[all]" --no-cache-dir
pip install docarray==0.21.0
jina
export JINA_LOG_LEVEL="ERROR"
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Test docker compose
run: |
pytest -v -s --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml ./tests/docker_compose/test_*.py
timeout-minutes: 30
- name: Check codecov file
id: check_files
uses: andstor/file-existence-action@v1
with:
files: "coverage.xml"
- name: Upload coverage from test to Codecov
uses: codecov/codecov-action@v3.1.1
if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8'
with:
file: coverage.xml
name: ${{ matrix.test-path }}-codecov
flags: ${{ steps.test.outputs.codecov_flag }}
fail_ci_if_error: false
prep-testbed:
runs-on: ubuntu-latest
needs: [ lint-flake-8, code-injection]
steps:
- uses: actions/checkout@v2.5.0
- id: set-matrix
run: |
sudo apt-get install jq
export value=$(bash scripts/get-all-test-paths.sh)
echo "matrix=$value" >> $GITHUB_OUTPUT
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
docarray-v-two-test:
needs: prep-testbed
runs-on: ubuntu-latest
env:
JINA_RANDOM_PORT_MIN: 16384
strategy:
fail-fast: false
matrix:
python-version: [3.8]
protobuf-version: ['==3.19.6', '']
steps:
- uses: actions/checkout@v2.5.0
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Set up Golang
uses: actions/setup-go@v2
with:
go-version: 1.19.5
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install setuptools wheel
pip install git+https://github.com/jina-ai/setuptools-golang.git@feat-align-with-alaeddine-code
pip install cibuildwheel
- name: Build wheels with setuptools-golang-build-manylinux-wheel
run: |
setuptools-golang-build-manylinux-wheels --pythons cp38-cp38
- name: Prepare environment
run: |
docker build -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip .
python -m pip install --upgrade pip
python -m pip install wheel
WHEEL_FILE=$(ls dist/*whl)
pip install "$WHEEL_FILE[common,devel,test]" --no-cache-dir
if [[ "${{ matrix.protobuf-version }}" == "==3.19.6" ]]; then
pip install -U protobuf${{ matrix.protobuf-version }} grpcio==1.47.5 grpcio-reflection==1.47.5 grpcio-health-checking==1.47.5
else
pip install -U protobuf${{ matrix.protobuf-version }}
fi
jina
export JINA_LOG_LEVEL="ERROR"
- name: Test
id: test
run: |
pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/unit/serve/runtimes/test_helper.py
pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/docarray_v2/test_v2.py
pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/deployment_http_composite
pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/docarray_v2/test_singleton.py
pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/docarray_v2/test_parameters_as_pydantic.py
pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/docarray_v2/test_streaming.py
pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/docarray_v2/csp
pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/docarray_v2/docker
echo "flag it as jina for codeoverage"
echo "codecov_flag=jina" >> $GITHUB_OUTPUT
timeout-minutes: 45
env:
JINA_AUTH_TOKEN: "${{ secrets.JINA_AUTH_TOKEN }}"
- name: Check codecov file
id: check_files
uses: andstor/file-existence-action@v1
with:
files: "coverage.xml"
- name: Upload coverage from test to Codecov
uses: codecov/codecov-action@v3.1.1
if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8'
with:
file: coverage.xml
name: ${{ matrix.test-path }}-codecov
flags: ${{ steps.test.outputs.codecov_flag }}
fail_ci_if_error: false
stateful-docarray-v-two-test:
needs: prep-testbed
runs-on: ubuntu-latest
env:
JINA_RANDOM_PORT_MIN: 16384
strategy:
fail-fast: false
matrix:
python-version: [3.8]
protobuf-version: ['==3.19.6', '']
steps:
- uses: actions/checkout@v2.5.0
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Set up Golang
uses: actions/setup-go@v2
with:
go-version: 1.19.5
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install setuptools wheel
pip install git+https://github.com/jina-ai/setuptools-golang.git@feat-align-with-alaeddine-code
pip install cibuildwheel
- name: Build wheels with setuptools-golang-build-manylinux-wheel
run: |
setuptools-golang-build-manylinux-wheels --pythons cp38-cp38
- name: Prepare environment
run: |
docker build -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip .
python -m pip install --upgrade pip
python -m pip install wheel
WHEEL_FILE=$(ls dist/*whl)
pip install "$WHEEL_FILE[common,devel,test]" --no-cache-dir
if [[ "${{ matrix.protobuf-version }}" == "==3.19.6" ]]; then
pip install -U protobuf${{ matrix.protobuf-version }} grpcio==1.47.5 grpcio-reflection==1.47.5 grpcio-health-checking==1.47.5
else
pip install -U protobuf${{ matrix.protobuf-version }}
fi
jina
export JINA_LOG_LEVEL="ERROR"
- name: Test stateful
id: test_stateful
run: |
JINA_LOG_LEVEL=DEBUG JINA_RANDOM_PORT_MAX="60535" pytest --suppress-no-test-exit-code --maxfail 1 --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/stateful
echo "flag it as jina for codeoverage"
echo "codecov_flag=jina" >> $GITHUB_OUTPUT
timeout-minutes: 30
env:
JINA_AUTH_TOKEN: "${{ secrets.JINA_AUTH_TOKEN }}"
- name: Check codecov file
id: check_files
uses: andstor/file-existence-action@v1
with:
files: "coverage.xml"
- name: Upload coverage from test to Codecov
uses: codecov/codecov-action@v3.1.1
if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8'
with:
file: coverage.xml
name: ${{ matrix.test-path }}-codecov
flags: ${{ steps.test.outputs.codecov_flag }}
fail_ci_if_error: false
# Build the wheels for Linux and macOS for Python 3.8 and newer
build-wheels:
runs-on: ${{ matrix.os }}
strategy:
# Ensure that a wheel builder finishes even if another fails
fail-fast: false
matrix:
include:
# linux
- os: ubuntu-latest
python: '3.7'
python-manylinux-tag: "cp37-cp37m"
- os: ubuntu-latest
python: '3.8'
python-manylinux-tag: "cp38-cp38"
- os: ubuntu-latest
python: '3.9'
python-manylinux-tag: "cp39-cp39"
- os: ubuntu-latest
python: '3.10'
python-manylinux-tag: "cp310-cp310"
- os: ubuntu-latest
python: '3.11'
python-manylinux-tag: "cp311-cp311"
# MacOS emulated
- os: macos-latest
python: '3.8'
python-cibuildwheels: '38'
platform_id: macosx_x86_64
arch: x86_64
goarch: amd64
- os: macos-latest
python: '3.9'
python-cibuildwheels: '39'
platform_id: macosx_x86_64
arch: x86_64
goarch: amd64
- os: macos-latest
python: '3.10'
python-cibuildwheels: '310'
platform_id: macosx_x86_64
arch: x86_64
goarch: amd64
- os: macos-latest
python: '3.11'
python-cibuildwheels: '311'
platform_id: macosx_x86_64
arch: x86_64
goarch: amd64
# MacOS native
- os: macos-latest
python: '3.8'
python-cibuildwheels: '38'
platform_id: macosx_arm64
arch: arm64
goarch: arm64
- os: macos-latest
python: '3.9'
python-cibuildwheels: '39'
platform_id: macosx_arm64
arch: arm64
goarch: arm64
- os: macos-latest
python: '3.10'
python-cibuildwheels: '310'
platform_id: macosx_arm64
arch: arm64
goarch: arm64
- os: macos-latest
python: '3.11'
python-cibuildwheels: '311'
platform_id: macosx_arm64
arch: arm64
goarch: arm64
steps:
- name: Check out repository
uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python }}
- name: Set up Golang
uses: actions/setup-go@v2
with:
go-version: 1.19.5
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install setuptools wheel
pip install git+https://github.com/jina-ai/setuptools-golang.git@feat-align-with-alaeddine-code
pip install cibuildwheel
- name: Build wheels with setuptools-golang-build-manylinux-wheel
if: ${{ matrix.os == 'ubuntu-latest' }}
run: |
setuptools-golang-build-manylinux-wheels --pythons ${{ matrix.python-manylinux-tag }}
- name: Build wheels with cibuildwheels on macos
env:
CIBW_BUILD: cp${{ matrix.python-cibuildwheels }}-${{ matrix.platform_id }}
CIBW_ARCHS: ${{ matrix.arch }}
CIBW_TEST_COMMAND: python -c "import jina; import jraft"
CIBW_BUILD_VERBOSITY: 1
GOARCH: ${{ matrix.goarch }}
CGO_ENABLED: 1
if: ${{ matrix.os == 'macos-latest' }}
run: |
python -m cibuildwheel --output-dir dist
- name: Upload wheels as artifacts
uses: actions/upload-artifact@v2
with:
path: dist/*.whl
core-test:
needs: [prep-testbed]
runs-on: ubuntu-latest
env:
JINA_RANDOM_PORT_MIN: 16384
strategy:
fail-fast: false
matrix:
python-version: [3.8]
test-path: ${{fromJson(needs.prep-testbed.outputs.matrix)}}
steps:
- uses: actions/checkout@v2.5.0
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Set up Golang
uses: actions/setup-go@v2
with:
go-version: 1.19.5
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install setuptools wheel
pip install git+https://github.com/jina-ai/setuptools-golang.git@feat-align-with-alaeddine-code
pip install cibuildwheel
- name: Build wheels with setuptools-golang-build-manylinux-wheel
run: |
setuptools-golang-build-manylinux-wheels --pythons cp38-cp38
- name: Prepare environment
run: |
docker build --build-arg DOCARRAY_VERSION="0.21.0" -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip .
python -m pip install --upgrade pip
python -m pip install wheel
WHEEL_FILE=$(ls dist/*whl)
pip install "$WHEEL_FILE[all]" --no-cache-dir
pip install docarray==0.21.0
jina
export JINA_LOG_LEVEL="ERROR"
- name: Test
id: test
run: |
pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/docarray_v2/*' --ignore-glob='tests/integration/stateful/*' --ignore-glob='tests/integration/hub_usage/dummyhub*' ${{ matrix.test-path }}
if [[ "${{ matrix.test-path }}" == *"tests/integration/instrumentation/" ]]
then
echo "sleeping to let open telemetry export daemon thread expire by itself"
sleep 2
fi
echo "flag it as jina for codeoverage"
echo "codecov_flag=jina" >> $GITHUB_OUTPUT
timeout-minutes: 30
env:
JINA_AUTH_TOKEN: "${{ secrets.JINA_AUTH_TOKEN }}"
- name: Check codecov file
id: check_files
uses: andstor/file-existence-action@v1
with:
files: "coverage.xml"
- name: Upload coverage from test to Codecov
uses: codecov/codecov-action@v3.1.1
if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8'
with:
file: coverage.xml
name: ${{ matrix.test-path }}-codecov
flags: ${{ steps.test.outputs.codecov_flag }}
fail_ci_if_error: false
import-test:
runs-on: ubuntu-latest
needs: [lint-flake-8, code-injection ]
strategy:
fail-fast: false
matrix:
core: ['', 'true']
perf: ['', 'true']
python-env: ['3.7', '3.8', '3.9', '3.10', '3.11']
exclude:
- core: 'true'
perf: 'true'
steps:
- uses: actions/checkout@v2.5.0
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Prepare enviroment
run: |
python -m pip install --upgrade pip
python -m pip install wheel
pip install --no-cache-dir .
env:
JINA_PIP_INSTALL_CORE: ${{ matrix.core }}
JINA_PIP_INSTALL_PERF: ${{ matrix.perf }}
- name: Test basic import
run: python -c 'from jina import Executor,requests,Client,Flow,dynamic_batching,Document,DocumentArray'
- name: Test import all
run: python -c 'from jina import *'
install-jina-ecosystem-test:
runs-on: ubuntu-latest
needs: [lint-flake-8, code-injection ]
strategy:
fail-fast: false
steps:
- uses: actions/checkout@v2.5.0
- name: Set up Python 3.8
uses: actions/setup-python@v4
with:
python-version: 3.8
- name: Prepare enviroment
run: |
python -m pip install --upgrade pip
python -m pip install wheel
HUBBLE_VERSION=$(curl -L -s "https://pypi.org/pypi/jina-hubble-sdk/json" | jq -r '.releases | keys | .[]| select(startswith("0."))' | sort -V | tail -1)
DOCARRAY_VERSION=$(curl -L -s "https://pypi.org/pypi/docarray/json" | jq -r '.releases | keys | .[]| select(startswith("0."))' | sort -V | tail -1)
JCLOUD_VERSION=$(curl -L -s "https://pypi.org/pypi/jcloud/json" | jq -r '.releases | keys | .[]| select(startswith("0."))' | sort -V | tail -1)
pip install . jcloud==$JCLOUD_VERSION docarray==$DOCARRAY_VERSION jina-hubble-sdk==$HUBBLE_VERSION
env:
JINA_PIP_INSTALL_CORE: ${{ matrix.core }}
JINA_PIP_INSTALL_PERF: ${{ matrix.perf }}
- name: Test basic import
run: python -c 'from jina import Executor,requests'
- name: Test import all
run: python -c 'from jina import *'
# just for blocking the merge until all parallel core-test are successful
success-all-test:
runs-on: ubuntu-latest
needs: [commit-lint, core-test, docarray-v-two-test, stateful-docarray-v-two-test, import-test, hub-test, k8s-flow-test, k8s-deployment-test, k8s-graceful-test, k8s-failures-test, k8s-otel-test, docker-compose-test, docker-image-test, check-docstring, check-black, code-injection, build-wheels]
if: always()
steps:
- uses: technote-space/workflow-conclusion-action@v2
- name: Check Failure
if: env.WORKFLOW_CONCLUSION == 'failure'
run: exit 1
- name: Success
if: ${{ success() }}
run: echo "All Done"