diff --git a/.github/workflows/check-plugin.yml b/.github/workflows/check-plugin.yml new file mode 100644 index 000000000..a32c0f651 --- /dev/null +++ b/.github/workflows/check-plugin.yml @@ -0,0 +1,134 @@ +name: Running tests and linter + +on: + workflow_call: + inputs: + plugin: + type: string + +jobs: + unit-tests: + defaults: + run: + shell: bash + strategy: + matrix: + os: [ ubuntu-latest, windows-latest ] + python-version: [ "3.7", "3.8", "3.9", "3.10" ] + runs-on: ${{ matrix.os }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up Python ${{matrix.python-version}} + uses: actions/setup-python@v3 + with: + python-version: ${{matrix.python-version}} + - name: Cache python packages for Linux + if: matrix.os == 'ubuntu-latest' + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{inputs.plugin}}-${{matrix.os}}-python-${{matrix.python-version}} + restore-keys: ${{inputs.plugin}} + - name: Cache python packages for Windows + if: matrix.os == 'windows-latest' + uses: actions/cache@v3 + with: + path: ~\AppData\Local\pip\Cache + key: ${{inputs.plugin}}-${{matrix.os}}-python-${{matrix.python-version}} + restore-keys: ${{inputs.plugin}} + - name: Install Kedro + run: pip install git+https://github.com/kedro-org/kedro@main + - name: Install dependencies + run: | + cd ${{ inputs.plugin }} + pip install -r test_requirements.txt + - name: Install pytables (only for kedro-datasets on windows) + if: matrix.os == 'windows-latest' && inputs.plugin == 'kedro-datasets' + run: pip install tables + - name: pip freeze + run: pip freeze + - name: Run unit tests for Linux / all plugins + if: matrix.os != 'windows-latest' + run: make plugin=${{ inputs.plugin }} test + - name: Run unit tests for Windows / kedro-airflow, kedro-docker, kedro-telemetry + if: matrix.os == 'windows-latest' && inputs.plugin != 'kedro-datasets' + run: | + cd ${{ inputs.plugin }} + pytest tests + - name: Run unit tests for Windows / kedro-datasets / no spark sequential + if: matrix.os == 'windows-latest' && inputs.plugin == 'kedro-datasets' && matrix.python-version == '3.10' + run: | + make test-no-spark-sequential + - name: Run unit tests for Windows / kedro-datasets / no spark parallel + if: matrix.os == 'windows-latest' && inputs.plugin == 'kedro-datasets' && matrix.python-version != '3.10' + run: | + make test-no-spark + + lint: + defaults: + run: + shell: bash + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up Python 3.8 + uses: actions/setup-python@v3 + with: + python-version: 3.8 + - name: Cache python packages + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{inputs.plugin}}-${{matrix.os}}-python-${{matrix.python-version}} + restore-keys: ${{inputs.plugin}} + - name: Install dependencies + run: | + cd ${{ inputs.plugin }} + pip install git+https://github.com/kedro-org/kedro@main + pip install -r test_requirements.txt + pip freeze + - name: Install pre-commit hooks + run: | + cd ${{ inputs.plugin }} + pre-commit install --install-hooks + pre-commit install --hook-type pre-push + - name: Run linter + run: make plugin=${{ inputs.plugin }} lint + + e2e-tests: + if: inputs.plugin != 'kedro-datasets' + defaults: + run: + shell: bash + strategy: + matrix: + os: [ ubuntu-latest ] + python-version: [ "3.7", "3.8", "3.9", "3.10" ] + runs-on: ${{ matrix.os }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up Python ${{matrix.python-version}} + uses: actions/setup-python@v3 + with: + python-version: ${{matrix.python-version}} + - name: Cache python packages + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{inputs.plugin}}-${{matrix.os}}-python-${{matrix.python-version}} + restore-keys: ${{inputs.plugin}} + - name: Install dependencies + run: | + cd ${{ inputs.plugin }} + pip install git+https://github.com/kedro-org/kedro@main + pip install -r test_requirements.txt + - name: pip freeze + run: pip freeze + - name: Run end to end tests + # Custom shell to run kedro-docker e2e-tests because -it flag for `docker run` + # isn't supported on Github Actions. See https://github.com/actions/runner/issues/241 + shell: 'script -q -e -c "bash {0}"' + run: make plugin=${{ inputs.plugin }} e2e-tests diff --git a/.github/workflows/kedro-airflow.yml b/.github/workflows/kedro-airflow.yml new file mode 100644 index 000000000..b68fcce30 --- /dev/null +++ b/.github/workflows/kedro-airflow.yml @@ -0,0 +1,16 @@ +name: Run checks on kedro-airflow + +on: + push: + paths: + - "kedro-airflow/**" + pull_request: + paths: + - "kedro-airflow/**" + types: [ synchronize ] + +jobs: + airflow-test: + uses: ./.github/workflows/check-plugin.yml + with: + plugin: kedro-airflow diff --git a/.github/workflows/kedro-datasets.yml b/.github/workflows/kedro-datasets.yml new file mode 100644 index 000000000..9ff4802b6 --- /dev/null +++ b/.github/workflows/kedro-datasets.yml @@ -0,0 +1,16 @@ +name: Run checks on kedro-datasets + +on: + push: + paths: + - "kedro-datasets/**" + pull_request: + paths: + - "kedro-datasets/**" + types: [ synchronize ] + +jobs: + datasets-test: + uses: ./.github/workflows/check-plugin.yml + with: + plugin: kedro-datasets diff --git a/.github/workflows/kedro-docker.yml b/.github/workflows/kedro-docker.yml new file mode 100644 index 000000000..1812a3a93 --- /dev/null +++ b/.github/workflows/kedro-docker.yml @@ -0,0 +1,16 @@ +name: Run checks on kedro-docker + +on: + push: + paths: + - "kedro-docker/**" + pull_request: + paths: + - "kedro-docker/**" + types: [ synchronize ] + +jobs: + docker-test: + uses: ./.github/workflows/check-plugin.yml + with: + plugin: kedro-docker diff --git a/.github/workflows/kedro-telemetry.yml b/.github/workflows/kedro-telemetry.yml new file mode 100644 index 000000000..fd75e8a71 --- /dev/null +++ b/.github/workflows/kedro-telemetry.yml @@ -0,0 +1,16 @@ +name: Run checks on kedro-telemetry + +on: + push: + paths: + - "kedro-telemetry/**" + pull_request: + paths: + - "kedro-telemetry/**" + types: [ synchronize ] + +jobs: + telemetry-test: + uses: ./.github/workflows/check-plugin.yml + with: + plugin: kedro-telemetry diff --git a/trufflehog-ignore.txt b/trufflehog-ignore.txt index 041fc7ffd..1929a2634 100644 --- a/trufflehog-ignore.txt +++ b/trufflehog-ignore.txt @@ -1 +1,3 @@ kedro-telemetry/README.md +kedro-telemetry/RELEASE.md +kedro-datasets/tests/tensorflow/test_tensorflow_model_dataset.py