diff --git a/.github/cloud/service.tf b/.github/cloud/service.tf index 45a4304951..1e7ec9e2e6 100644 --- a/.github/cloud/service.tf +++ b/.github/cloud/service.tf @@ -2,7 +2,7 @@ terraform { required_providers { clickhouse = { source = "ClickHouse/clickhouse" - version = "0.0.6" + version = "~> 1.2" } } } @@ -50,6 +50,7 @@ resource "clickhouse_service" "service" { region = "us-east-2" tier = "development" password = var.service_password + idle_scaling = false ip_access = [ { diff --git a/.github/workflows/run-tests-cloud.yml b/.github/workflows/run-tests-cloud.yml index 2069dfbb7b..883f96dc5f 100644 --- a/.github/workflows/run-tests-cloud.yml +++ b/.github/workflows/run-tests-cloud.yml @@ -1,4 +1,4 @@ -name: run-tests-cloud +name: Tests cloud permissions: write-all @@ -10,88 +10,62 @@ on: types: [opened, labeled, synchronize, reopened] jobs: - cloud: + test: if: github.event_name == 'push' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'tests:run-cloud')) - runs-on: ubuntu-latest + runs-on: [self-hosted, style-checker-aarch64] defaults: run: shell: bash strategy: - fail-fast: true - max-parallel: 1 matrix: go: - "1.23" steps: - - name: Check Out Code - uses: actions/checkout@v3 + - name: Checkout + uses: actions/checkout@v4 - - name: Resolve runner IP - run: echo "TF_VAR_allowed_cidr=$(curl -s ifconfig.me)/32" >> $GITHUB_ENV + - name: Install Go ${{ matrix.go }} + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go }} - name: Setup Terraform - uses: hashicorp/setup-terraform@v2.0.3 + uses: hashicorp/setup-terraform@v3 with: - terraform_version: 1.3.4 terraform_wrapper: false - - name: Terraform Init - id: init - working-directory: .github/cloud/ - run: terraform init - - - name: Terraform Validate - id: validate - working-directory: .github/cloud/ - run: terraform validate -no-color - - - name: Set service name for pull request - if: github.event_name == 'pull_request' - working-directory: .github/cloud/ - run: echo "TF_VAR_service_name=clickhouse-go ${{ matrix.go }} PR${{ github.event.pull_request.number }} $(date +'%Y%m%d%H%M%S')" >> $GITHUB_ENV - - - name: Set service name for push - if: github.event_name == 'push' + - name: Create Cloud instance + id: create working-directory: .github/cloud/ - run: echo "TF_VAR_service_name=clickhouse-go ${{ matrix.go }} $(git rev-parse --short HEAD) $(date +'%Y%m%d%H%M%S')" >> $GITHUB_ENV - - - name: Terraform Apply - working-directory: .github/cloud/ - id: apply - run: terraform apply -no-color -auto-approve -input=false -lock=false + run: | + export TF_VAR_allowed_cidr="$(curl -s ifconfig.me)/32" + export TF_VAR_service_name="clickhouse-go $(git rev-parse --short HEAD) $(date +'%Y%m%d%H%M%S')" + terraform init + terraform validate + terraform apply -auto-approve -input=false -lock=false + terraform output + echo "CLICKHOUSE_HOST=$(terraform output -raw CLICKHOUSE_HOST)" >> $GITHUB_ENV env: TF_VAR_organization_id: ${{ secrets.INTEGRATIONS_TEAM_TESTS_ORGANIZATION_ID }} - TF_VAR_token_key: ${{ secrets.INTEGRATIONS_TEAM_TESTS_TOKEN_KEY }} - TF_VAR_token_secret: ${{ secrets.INTEGRATIONS_TEAM_TESTS_TOKEN_SECRET }} + TF_VAR_token_key: ${{ secrets.INTEGRATIONS_TEAM_TESTS_TOKEN_KEY }} + TF_VAR_token_secret: ${{ secrets.INTEGRATIONS_TEAM_TESTS_TOKEN_SECRET }} TF_VAR_service_password: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_PASSWORD }} TF_VAR_api_url: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_API_URL }} - - name: Set Host - working-directory: .github/cloud/ - run: echo "CLICKHOUSE_HOST=$(terraform output -raw CLICKHOUSE_HOST)" >> $GITHUB_ENV - - - name: Service ID - working-directory: .github/cloud/ - run: terraform output -raw SERVICE_ID - - - name: Install Go ${{ matrix.go }} - uses: actions/setup-go@v2.1.5 - with: - stable: false - go-version: ${{ matrix.go }} - - - name: Run tests + - name: Tests env: CLICKHOUSE_PASSWORD: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_PASSWORD }} CLICKHOUSE_USE_DOCKER: false CLICKHOUSE_USE_SSL: true + TESTCONTAINERS_RYUK_DISABLED: "true" run: | CLICKHOUSE_DIAL_TIMEOUT=20 CLICKHOUSE_TEST_TIMEOUT=600s CLICKHOUSE_QUORUM_INSERT=2 make test - - name: Cleanup + - name: Cleanup Cloud instance if: always() working-directory: .github/cloud/ - run: terraform destroy -no-color -auto-approve -input=false -lock=false + run: | + terraform destroy -no-color -auto-approve -input=false -lock=false env: TF_VAR_organization_id: ${{ secrets.INTEGRATIONS_TEAM_TESTS_ORGANIZATION_ID }} TF_VAR_token_key: ${{ secrets.INTEGRATIONS_TEAM_TESTS_TOKEN_KEY }} diff --git a/.github/workflows/run-tests-head.yml b/.github/workflows/run-tests-head.yml index ad70cd3861..26ec35a724 100644 --- a/.github/workflows/run-tests-head.yml +++ b/.github/workflows/run-tests-head.yml @@ -1,4 +1,4 @@ -name: run-tests-head +name: Tests head permissions: write-all @@ -10,24 +10,26 @@ on: types: [opened, labeled, synchronize, reopened] jobs: - test-ch-head: + test: if: github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'tests:run-head')) - runs-on: ubuntu-latest + runs-on: [self-hosted, style-checker-aarch64] strategy: - fail-fast: true - max-parallel: 1 matrix: go: - "1.23" + clickhouse: + - "head" steps: - - uses: actions/checkout@main + - name: Checkout + uses: actions/checkout@v4 - name: Install Go ${{ matrix.go }} - uses: actions/setup-go@v2.1.5 + uses: actions/setup-go@v5 with: - stable: false go-version: ${{ matrix.go }} - - name: Run tests + - name: Tests + env: + TESTCONTAINERS_RYUK_DISABLED: "true" run: | - CLICKHOUSE_VERSION=head make test + CLICKHOUSE_VERSION=${{ matrix.clickhouse }} make test diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index fddabedcfa..9d691d571d 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -1,4 +1,4 @@ -name: run-tests +name: Tests matrix permissions: write-all @@ -11,30 +11,29 @@ on: - main jobs: - single-node: - runs-on: ubuntu-latest + test: + runs-on: [self-hosted, style-checker-aarch64] strategy: - fail-fast: true - max-parallel: 1 matrix: go: - "1.23" - "1.22" - - "1.21" clickhouse: # https://github.com/ClickHouse/ClickHouse/blob/master/SECURITY.md#scope-and-supported-versions + - "24.9" - "24.8" - "24.7" - - "24.6" - "24.3" steps: - - uses: actions/checkout@main + - name: Checkout + uses: actions/checkout@v4 - name: Install Go ${{ matrix.go }} - uses: actions/setup-go@v2.1.5 + uses: actions/setup-go@v5 with: - stable: false go-version: ${{ matrix.go }} - - name: Run tests + - name: Tests + env: + TESTCONTAINERS_RYUK_DISABLED: "true" run: | CLICKHOUSE_VERSION=${{ matrix.clickhouse }} make test diff --git a/Makefile b/Makefile index acd3a74f00..28c63515ba 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ CLICKHOUSE_VERSION ?= latest -CLICKHOUSE_TEST_TIMEOUT ?= 120s +CLICKHOUSE_TEST_TIMEOUT ?= 240s CLICKHOUSE_QUORUM_INSERT ?= 1 up: diff --git a/tests/issues/1421_test.go b/tests/issues/1421_test.go index 904ee4fde7..da21ec2a61 100644 --- a/tests/issues/1421_test.go +++ b/tests/issues/1421_test.go @@ -18,6 +18,8 @@ import ( const insertQry = "INSERT INTO test (foo, foo2)" func Test1421BatchFlushBrokenConn(t *testing.T) { + tests.SkipOnCloud(t, "This test requires container environment") + // create a dedicated test environment for this test // note: test environment management is a bit messy, consider refactoring env, err := tests.CreateClickHouseTestEnvironment(t.Name())