Migrate ocisIntegrationTests and s3ngIntegrationTests to GitHub Actio…

…ns (cs3org#3640) * Migrate ocisIntegrationTests and s3ngIntegrationTests to GitHub Actions * Temporary fix
gmgigi96 · Jun 28, 2023 · f6aa236 · f6aa236
1 parent 2954ce5
commit f6aa236
Show file tree

Hide file tree

Showing 31 changed files with 727 additions and 461 deletions.
diff --git a/.drone.star b/.drone.star
@@ -62,92 +62,80 @@ def cephService():
 
 # Pipeline definitions
 def main(ctx):
-    # In order to run specific parts only, specify the parts as
-    # ocisIntegrationTests(6, [1, 4])     - this will only run 1st and 4th parts
-    # implemented for: ocisIntegrationTests and s3ngIntegrationTests
     return [
         checkStarlark(),
-    ] + ocisIntegrationTests(6) + s3ngIntegrationTests(12)
+        ocisIntegrationTest(),
+    ] + s3ngIntegrationTests()
 
-def ocisIntegrationTests(parallelRuns, skipExceptParts = []):
-    pipelines = []
-    debugPartsEnabled = (len(skipExceptParts) != 0)
-    for runPart in range(1, parallelRuns + 1):
-        if debugPartsEnabled and runPart not in skipExceptParts:
-            continue
-
-        pipelines.append(
+def ocisIntegrationTest():
+    return {
+        "kind": "pipeline",
+        "type": "docker",
+        "name": "ocis-integration-tests-2",
+        "platform": {
+            "os": "linux",
+            "arch": "amd64",
+        },
+        "trigger": {
+            "event": {
+                "include": [
+                    "pull_request",
+                    "tag",
+                ],
+            },
+        },
+        "steps": [
+            makeStep(),
             {
-                "kind": "pipeline",
-                "type": "docker",
-                "name": "ocis-integration-tests-%s" % runPart,
-                "platform": {
-                    "os": "linux",
-                    "arch": "amd64",
-                },
-                "trigger": {
-                    "event": {
-                        "include": [
-                            "pull_request",
-                            "tag",
-                        ],
-                    },
-                },
-                "steps": [
-                    makeStep(),
-                    {
-                        "name": "revad-services",
-                        "image": OC_CI_GOLANG,
-                        "detach": True,
-                        "commands": [
-                            "cd /drone/src/tests/oc-integration-tests/drone/",
-                            "/drone/src/cmd/revad/revad -c frontend.toml &",
-                            "/drone/src/cmd/revad/revad -c gateway.toml &",
-                            "/drone/src/cmd/revad/revad -c shares.toml &",
-                            "/drone/src/cmd/revad/revad -c storage-home-ocis.toml &",
-                            "/drone/src/cmd/revad/revad -c storage-users-ocis.toml &",
-                            "/drone/src/cmd/revad/revad -c storage-publiclink.toml &",
-                            "/drone/src/cmd/revad/revad -c ldap-users.toml",
-                        ],
-                    },
-                    cloneApiTestReposStep(),
-                    {
-                        "name": "APIAcceptanceTestsOcisStorage",
-                        "image": OC_CI_PHP,
-                        "commands": [
-                            "cd /drone/src/tmp/testrunner",
-                            "make test-acceptance-from-core-api",
-                        ],
-                        "environment": {
-                            "TEST_SERVER_URL": "http://revad-services:20080",
-                            "OCIS_REVA_DATA_ROOT": "/drone/src/tmp/reva/data/",
-                            "DELETE_USER_DATA_CMD": "rm -rf /drone/src/tmp/reva/data/nodes/root/* /drone/src/tmp/reva/data/nodes/*-*-*-* /drone/src/tmp/reva/data/blobs/*",
-                            "STORAGE_DRIVER": "OCIS",
-                            "SKELETON_DIR": "/drone/src/tmp/testing/data/apiSkeleton",
-                            "TEST_WITH_LDAP": "true",
-                            "REVA_LDAP_HOSTNAME": "ldap",
-                            "TEST_REVA": "true",
-                            "SEND_SCENARIO_LINE_REFERENCES": "true",
-                            "BEHAT_FILTER_TAGS": "~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage&&~@personalSpace&&~@issue-ocis-3023&&~@skipOnGraph&&~@caldav&&~@carddav&&~@skipOnReva",
-                            "DIVIDE_INTO_NUM_PARTS": parallelRuns,
-                            "RUN_PART": runPart,
-                            "EXPECTED_FAILURES_FILE": "/drone/src/tests/acceptance/expected-failures-on-OCIS-storage.md",
-                        },
-                    },
+                "name": "revad-services",
+                "image": OC_CI_GOLANG,
+                "detach": True,
+                "commands": [
+                    "cd /drone/src/tests/oc-integration-tests/drone/",
+                    "/drone/src/cmd/revad/revad -c frontend.toml &",
+                    "/drone/src/cmd/revad/revad -c gateway.toml &",
+                    "/drone/src/cmd/revad/revad -c shares.toml &",
+                    "/drone/src/cmd/revad/revad -c storage-home-ocis.toml &",
+                    "/drone/src/cmd/revad/revad -c storage-users-ocis.toml &",
+                    "/drone/src/cmd/revad/revad -c storage-publiclink.toml &",
+                    "/drone/src/cmd/revad/revad -c ldap-users.toml",
                 ],
-                "services": [
-                    ldapService(),
+            },
+            cloneApiTestReposStep(),
+            {
+                "name": "APIAcceptanceTestsOcisStorage",
+                "image": OC_CI_PHP,
+                "commands": [
+                    "cd /drone/src/tmp/testrunner",
+                    "make test-acceptance-from-core-api",
                 ],
+                "environment": {
+                    "TEST_SERVER_URL": "http://revad-services:20080",
+                    "OCIS_REVA_DATA_ROOT": "/drone/src/tmp/reva/data/",
+                    "DELETE_USER_DATA_CMD": "rm -rf /drone/src/tmp/reva/data/nodes/root/* /drone/src/tmp/reva/data/nodes/*-*-*-* /drone/src/tmp/reva/data/blobs/*",
+                    "STORAGE_DRIVER": "OCIS",
+                    "SKELETON_DIR": "/drone/src/tmp/testing/data/apiSkeleton",
+                    "TEST_WITH_LDAP": "true",
+                    "REVA_LDAP_HOSTNAME": "ldap",
+                    "TEST_REVA": "true",
+                    "SEND_SCENARIO_LINE_REFERENCES": "true",
+                    "BEHAT_FILTER_TAGS": "~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage&&~@personalSpace&&~@issue-ocis-3023&&~@skipOnGraph&&~@caldav&&~@carddav&&~@skipOnReva",
+                    "DIVIDE_INTO_NUM_PARTS": 6,
+                    "RUN_PART": 2,
+                    "EXPECTED_FAILURES_FILE": "/drone/src/tests/acceptance/expected-failures-on-OCIS-storage.md",
+                },
             },
-        )
-
-    return pipelines
+        ],
+        "services": [
+            ldapService(),
+        ],
+    }
 
-def s3ngIntegrationTests(parallelRuns, skipExceptParts = []):
+def s3ngIntegrationTests():
+    parallelRuns = 12
     pipelines = []
-    debugPartsEnabled = (len(skipExceptParts) != 0)
     for runPart in range(1, parallelRuns + 1):
-        if debugPartsEnabled and runPart not in skipExceptParts:
+        if runPart in [9]:
             continue
 
         pipelines.append(

diff --git a/.github/workflows/compose.yml b/.github/workflows/compose.yml
@@ -0,0 +1,37 @@
+name: Compose
+on:
+  workflow_call:
+    inputs:
+      test:
+        required: true
+        type: string
+      submodules:
+        type: boolean
+      parts:
+        type: number
+      part:
+        type: number
+
+jobs:
+  compose:
+    runs-on: self-hosted
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: ${{ inputs.submodules }}
+      - name: Download image
+        uses: ishworkh/docker-image-artifact-download@v1
+        with:
+          image: revad:test
+      - name: Test
+        uses: isbang/compose-action@v1.4.1
+        with:
+          compose-file: ./tests/docker/docker-compose.yml
+          up-flags: --force-recreate --always-recreate-deps --build --abort-on-container-exit -V --remove-orphans --exit-code-from ${{ inputs.test }}
+          down-flags: --rmi all -v --remove-orphans
+          services: ${{ inputs.test }}
+        env:
+          REVAD_IMAGE: revad:test
+          PARTS: ${{ inputs.parts }}
+          PART: ${{ inputs.part }}
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -11,8 +11,6 @@ on:
         type: boolean
       push:
         type: boolean
-      build-args:
-        type: string
       platforms:
         type: string
 
@@ -28,7 +26,7 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
       - name: Login to Docker Hub
-        if: ${{ inputs.push }}
+        if: inputs.push
         uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
@@ -41,11 +39,10 @@ jobs:
           tags: ${{ inputs.tags }}
           load: ${{ inputs.load }}
           push: ${{ inputs.push }}
-          build-args: ${{ inputs.build-args }}
           platforms: ${{ inputs.platforms }}
       - name: Upload ${{ inputs.tags }} Docker image to artifacts
         uses: ishworkh/docker-image-artifact-upload@v1
-        if: ${{ inputs.load }}
+        if: inputs.load
         with:
           image: ${{ inputs.tags }}
           retention_days: '1'
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -58,34 +58,42 @@ jobs:
       file: docker/Dockerfile.revad
       tags: revad:test
       load: true
-  test:
+  litmus:
     needs: docker-revad
-    runs-on: self-hosted
     strategy:
       fail-fast: false
       matrix:
-        test: [litmus-1, litmus-2, litmus-3, acceptance-1]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Checkout core
-        uses: actions/checkout@v3
-        if: matrix.test == 'acceptance-1'
-        with:
-          repository: owncloud/ocis
-          ref: ${{ vars.OCIS_COMMITID }}
-          path: tests/ocis
-      - name: Checkout testing
-        uses: actions/checkout@v3
-        if: matrix.test == 'acceptance-1'
-        with:
-          repository: owncloud/testing
-          path: tests/testing
-      - name: Download image
-        uses: ishworkh/docker-image-artifact-download@v1
-        with:
-          image: revad:test
-      - name: Test
-        run: make ${{ matrix.test }} -o test-docker
-        env:
-          REVAD_IMAGE: revad:test
+        test: [litmus-1, litmus-2, litmus-3]
+    uses: ./.github/workflows/compose.yml
+    with:
+      test: ${{ matrix.test }}
+  acceptance-1:
+    uses: ./.github/workflows/compose.yml
+    with:
+      test: acceptance-1
+      submodules: true
+    needs: docker-revad
+  acceptance-2:
+    needs: docker-revad
+    strategy:
+      fail-fast: false
+      matrix:
+        part: [1, 3, 4, 5, 6]
+    uses: ./.github/workflows/compose.yml
+    with:
+      test: acceptance-2
+      submodules: true
+      parts: 6
+      part: ${{ matrix.part }}
+  acceptance-3:
+    needs: docker-revad
+    strategy:
+      fail-fast: false
+      matrix:
+        part: [9]
+    uses: ./.github/workflows/compose.yml
+    with:
+      test: acceptance-3
+      submodules: true
+      parts: 12
+      part: ${{ matrix.part }}
diff --git a/.gitignore b/.gitignore
@@ -35,19 +35,15 @@ docs/public/
 docs/resources/
 docs/node_modules/
 docs/tech-doc-hugo
-docs/themes/
 
 dist/
 bin/
 
 # API acceptance tests
-composer.lock
-/vendor
-/vendor-php
-vendor-bin/**/vendor
-vendor-bin/**/composer.lock
 tests/acceptance/output
 tmp/
+tests/ocis/tests/acceptance/tus-upload-test-*
+tests/ocis/tests/acceptance/work_tmp
 
 # drone
 .drone.yml

diff --git a/.gitmodules b/.gitmodules
@@ -1,9 +1,9 @@
-[submodule "docs/themes/docsy"]
-	path = docs/themes/docsy
-	url = https://github.com/google/docsy
 [submodule "tests/ocis"]
 	path = tests/ocis
 	url = https://github.com/owncloud/ocis.git
 [submodule "tests/testing"]
 	path = tests/testing
 	url = https://github.com/owncloud/testing.git
+[submodule "docs/themes/docsy"]
+	path = docs/themes/docsy
+	url = https://github.com/google/docsy.git
diff --git a/Makefile b/Makefile
@@ -38,34 +38,43 @@ BUILD_FLAGS	= "-X main.gitCommit=$(GIT_COMMIT) -X main.version=$(VERSION) -X mai
 revad:
 	go build -ldflags $(BUILD_FLAGS) -o ./cmd/revad/revad ./cmd/revad
 
-.PHONY: revad-cephfs
-revad-cephfs:
+.PHONY: revad-ceph
+revad-ceph:
 	go build -ldflags $(BUILD_FLAGS) -tags ceph -o ./cmd/revad/revad ./cmd/revad
 
 .PHONY: reva
 reva:
 	go build -ldflags $(BUILD_FLAGS) -o ./cmd/reva/reva ./cmd/reva
 
+.PHONY: docker-reva
+docker-reva:
+	docker build -f docker/Dockerfile.reva -t reva .
+
+.PHONY: docker-revad
+docker-revad:
+	docker build -f docker/Dockerfile.revad -t revad .
+
+.PHONY: docker-revad-ceph
+docker-revad-ceph:
+	docker build -f docker/Dockerfile.revad-ceph -t revad-ceph .
+
+.PHONY: docker-revad-eos
+docker-revad-eos:
+	docker build -f docker/Dockerfile.revad-eos -t revad-eos .
+
 ################################################################################
 # Test
 ################################################################################
 
-REVAD_IMAGE	?= revad:test
+TEST				= litmus-1 litmus-2 litmus-3 acceptance-1 acceptance-2 acceptance-3
+export REVAD_IMAGE	?= revad
+export PARTS		?= 1
+export PART			?= 1
 
-.PHONY: test-docker
-test-docker:
-	docker build -f docker/Dockerfile.revad -t $(REVAD_IMAGE) .
-
-TESTS		= litmus-1 litmus-2 litmus-3 acceptance-1
-TIMEOUT		?= 3600
-
-.PHONY: $(TESTS)
-$(TESTS): test-docker
-	docker pull cs3org/behat:latest
-	REVAD_IMAGE=$(REVAD_IMAGE) \
-	docker-compose --file tests/docker-compose/$@.yml --project-directory . \
-		up --force-recreate --renew-anon-volumes --remove-orphans \
-		--exit-code-from $@ --abort-on-container-exit --timeout $(TIMEOUT)
+.PHONY: $(TEST)
+$(TEST): docker-revad
+	docker-compose -f ./tests/docker/docker-compose.yml up --force-recreate --always-recreate-deps --build --abort-on-container-exit -V --remove-orphans --exit-code-from $@ $@; \
+	docker-compose -f ./tests/docker/docker-compose.yml down --rmi all -v --remove-orphans
 
 .PHONY: test-go
 test-go:
@@ -117,7 +126,3 @@ toolchain-clean:
 .PHONY: clean
 clean: toolchain-clean
 	rm -rf dist
-
-
-test-acceptance-api:
-	$(PATH_TO_APITESTS)/tests/acceptance/run.sh --type api