engine: fix dataflow engine build (pingcap#5352)

.github/workflows/check_and_build.yaml

@@ -29,6 +29,9 @@ jobs:
       - name: Build-dm
         run: docker build -f ./dm/Dockerfile .
 
+      - name: Build-engine
+        run: docker build -f ./engine/Dockerfile .
+
   mac_build:
     name: Mac OS Build
     runs-on: macos-latest

.github/workflows/dataflow_engine_e2e.yaml

@@ -0,0 +1,93 @@
+name: Dataflow Engine e2e tests
+
+# Controls when the action will run. Triggers the workflow on push or pull request
+# events but only for the master branch
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+    paths:
+      - 'engine/**'
+
+# See: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#concurrency.
+concurrency:
+  group: ${{ github.ref }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  Basic-workflow:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.18
+
+      - name: Build images
+        run: $GITHUB_WORKSPACE/engine/sample/prepare.sh
+
+      - name: Run containers
+        run: docker-compose -f $GITHUB_WORKSPACE/engine/sample/3m3e.yaml -f $GITHUB_WORKSPACE/engine/sample/demo.yaml up -d
+
+      - name: Run tests
+        run: |
+          cd $GITHUB_WORKSPACE/engine/test/e2e
+          go test -count=1 -v -run=TestSubmitTest
+      - name: Try to dump container logs
+        if: ${{ failure() }}
+        run: docker-compose -f $GITHUB_WORKSPACE/engine/sample/3m3e.yaml logs -t
+
+  Node-failure-workflow:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.18
+
+      - name: Build images
+        run: $GITHUB_WORKSPACE/engine/sample/prepare.sh
+
+      - name: Run containers
+        run: docker-compose -f $GITHUB_WORKSPACE/engine/sample/3m3e.yaml up -d
+
+      - name: Run tests
+        run: |
+          cd $GITHUB_WORKSPACE/engine/test/e2e
+          go test -count=1 -v -run=TestNodeFailure
+      - name: Try to dump container logs
+        if: ${{ failure() }}
+        run: docker-compose -f $GITHUB_WORKSPACE/engine/sample/3m3e.yaml logs -t
+
+  DM-workflow:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.18
+
+      - name: Build images
+        run: $GITHUB_WORKSPACE/engine/sample/prepare.sh
+
+      - name: Run containers
+        run: docker-compose -f $GITHUB_WORKSPACE/engine/sample/3m3e.yaml -f $GITHUB_WORKSPACE/engine/sample/dm_databases.yaml up -d
+
+      - name: Run tests
+        run: |
+          $GITHUB_WORKSPACE/engine/test/utils/wait_mysql_online.sh --host 127.0.0.1 --port 3306 --password 123456
+          $GITHUB_WORKSPACE/engine/test/utils/wait_mysql_online.sh --host 127.0.0.1 --port 4000
+          cd $GITHUB_WORKSPACE/engine/test/e2e
+          go test -count=1 -v -run=TestDMJob
+      - name: Try to dump container logs
+        if: ${{ failure() }}
+        run: |
+          docker-compose -f $GITHUB_WORKSPACE/engine/sample/3m3e.yaml logs -t
+          docker-compose -f $GITHUB_WORKSPACE/engine/sample/dm_databases.yaml logs -t

.github/workflows/dataflow_engine_ut.yaml

@@ -0,0 +1,35 @@
+name: Dataflow engine unit test
+
+env:
+  PROTOC_VERSION: 3.8.0
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+    paths:
+      - 'engine/**'
+
+# See: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#concurrency.
+concurrency:
+  group: ${{ github.ref }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  unit_test:
+    name: Make engine_unit_test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Setup Go environment
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.18
+
+      - name: Test
+        run: make engine_unit_test

.github/workflows/ticdc_integration.yaml

@@ -8,11 +8,13 @@ on:
       - master
     paths-ignore:
       - 'dm/**'
+      - 'engine/**'
   pull_request:
     branches:
       - master
     paths-ignore:
       - 'dm/**'
+      - 'engine/**'
 
 # See: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#concurrency.
 concurrency:

.golangci.yml

@@ -48,7 +48,7 @@ linters-settings:
       - name: var-naming
       - name: package-comments
       - name: range
-      - name: receiver-naming
+        # - name: receiver-naming TODO: disable it because it can't handle generics
       - name: indent-error-flow
       - name: superfluous-else
       - name: modifies-parameter

Makefile

@@ -1,7 +1,8 @@
 ### Makefile for ticdc
 .PHONY: build test check clean fmt cdc kafka_consumer coverage \
 	integration_test_build integration_test integration_test_mysql integration_test_kafka bank \
-	dm dm-master dm-worker dmctl dm-syncer dm_coverage
+	dm dm-master dm-worker dmctl dm-syncer dm_coverage \
+	engine df-master df-executor df-master-client df-demo df-chaos-case
 
 PROJECT=tiflow
 P=3
@@ -16,6 +17,7 @@ SHELL := /usr/bin/env bash
 
 TEST_DIR := /tmp/tidb_cdc_test
 DM_TEST_DIR := /tmp/dm_test
+ENGINE_TEST_DIR := /tmp/engine_test
 
 GO       := GO111MODULE=on go
 ifeq (${CDC_ENABLE_VENDOR}, 1)
@@ -33,11 +35,13 @@ MAC   := "Darwin"
 CDC_PKG := github.com/pingcap/tiflow
 DM_PKG := github.com/pingcap/tiflow/dm
 PACKAGE_LIST := go list ./... | grep -vE 'vendor|proto|tiflow\/tests|integration|testing_utils|pb|pbmock|tiflow\/bin'
-PACKAGE_LIST_WITHOUT_DM := $(PACKAGE_LIST) | grep -vE 'github.com/pingcap/tiflow/dm'
+PACKAGE_LIST_WITHOUT_DM_ENGINE := $(PACKAGE_LIST) | grep -vE 'github.com/pingcap/tiflow/dm|github.com/pingcap/tiflow/engine'
 DM_PACKAGE_LIST := go list github.com/pingcap/tiflow/dm/... | grep -vE 'pb|pbmock|dm/cmd'
 PACKAGES := $$($(PACKAGE_LIST))
-PACKAGES_WITHOUT_DM := $$($(PACKAGE_LIST_WITHOUT_DM))
+PACKAGES_TICDC := $$($(PACKAGE_LIST_WITHOUT_DM_ENGINE))
 DM_PACKAGES := $$($(DM_PACKAGE_LIST))
+ENGINE_PACKAGE_LIST := go list github.com/pingcap/tiflow/engine/... | grep -vE 'pb|proto|engine/cmd|engine/test/e2e'
+ENGINE_PACKAGES := $$($(ENGINE_PACKAGE_LIST))
 FILES := $$(find . -name '*.go' -type f | grep -vE 'vendor|kv_gen|proto|pb\.go|pb\.gw\.go')
 TEST_FILES := $$(find . -name '*_test.go' -type f | grep -vE 'vendor|kv_gen|integration|testing_utils')
 TEST_FILES_WITHOUT_DM := $$(find . -name '*_test.go' -type f | grep -vE 'vendor|kv_gen|integration|testing_utils|^\./dm')
@@ -91,9 +95,9 @@ all: dev install
 
 dev: check test
 
-test: unit_test dm_unit_test
+test: unit_test dm_unit_test engine_unit_test
 
-build: cdc dm
+build: cdc dm engine
 
 bank:
 	$(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/bank ./tests/bank/bank.go ./tests/bank/case.go
@@ -116,7 +120,7 @@ unit_test: check_failpoint_ctl generate_mock generate-msgp-code generate-protobu
 	mkdir -p "$(TEST_DIR)"
 	$(FAILPOINT_ENABLE)
 	@export log_level=error;\
-	$(GOTEST) -cover -covermode=atomic -coverprofile="$(TEST_DIR)/cov.unit.out" $(PACKAGES_WITHOUT_DM) \
+	$(GOTEST) -cover -covermode=atomic -coverprofile="$(TEST_DIR)/cov.unit.out" $(PACKAGES_TICDC) \
 	|| { $(FAILPOINT_DISABLE); exit 1; }
 	$(FAILPOINT_DISABLE)
 
@@ -125,7 +129,7 @@ unit_test_in_verify_ci: check_failpoint_ctl tools/bin/gotestsum tools/bin/gocov
 	$(FAILPOINT_ENABLE)
 	@export log_level=error;\
 	CGO_ENABLED=1 tools/bin/gotestsum --junitfile cdc-junit-report.xml -- -v -timeout 5m -p $(P) --race \
-	-covermode=atomic -coverprofile="$(TEST_DIR)/cov.unit.out" $(PACKAGES_WITHOUT_DM) \
+	-covermode=atomic -coverprofile="$(TEST_DIR)/cov.unit.out" $(PACKAGES_TICDC) \
 	|| { $(FAILPOINT_DISABLE); exit 1; }
 	tools/bin/gocov convert "$(TEST_DIR)/cov.unit.out" | tools/bin/gocov-xml > cdc-coverage.xml
 	$(FAILPOINT_DISABLE)
@@ -134,7 +138,7 @@ unit_test_in_verify_ci: check_failpoint_ctl tools/bin/gotestsum tools/bin/gocov
 leak_test: check_failpoint_ctl
 	$(FAILPOINT_ENABLE)
 	@export log_level=error;\
-	$(GOTEST) -count=1 --tags leak $(PACKAGES_WITHOUT_DM) || { $(FAILPOINT_DISABLE); exit 1; }
+	$(GOTEST) -count=1 --tags leak $(PACKAGES_TICDC) || { $(FAILPOINT_DISABLE); exit 1; }
 	$(FAILPOINT_DISABLE)
 
 check_third_party_binary:
@@ -466,3 +470,43 @@ failpoint-enable: check_failpoint_ctl
 
 failpoint-disable: check_failpoint_ctl
 	$(FAILPOINT_DISABLE)
+
+engine: df-master df-executor df-master-client df-demo
+
+df-proto:
+	./engine/generate-proto.sh
+
+df-master:
+	$(GOBUILD) -o bin/df-master ./engine/cmd/master
+	cp ./bin/df-master ./engine/ansible/roles/common/files/master.bin
+
+df-executor:
+	$(GOBUILD) -o bin/df-executor ./engine/cmd/executor
+	cp ./bin/df-executor ./engine/ansible/roles/common/files/executor.bin
+
+df-master-client:
+	$(GOBUILD) -o bin/df-master-client ./engine/cmd/master-client
+
+df-demo:
+	$(GOBUILD) -o bin/df-demoserver ./engine/cmd/demoserver
+	cp ./bin/df-demoserver ./engine/ansible/roles/common/files/demoserver.bin
+
+df-chaos-case:
+	$(GOBUILD) -o bin/df-chaos-case ./engine/chaos/cases
+
+df-kvmock: tools/bin/mockgen tools/bin/protoc tools/bin/protoc-gen-gogofaster
+	tools/bin/mockgen -package mock github.com/pingcap/tiflow/engine/pkg/meta/metaclient KVClient \
+	> engine/pkg/meta/kvclient/mock/mockclient.go
+
+engine_unit_test: check_failpoint_ctl
+	$(call run_engine_unit_test,$(ENGINE_PACKAGES))
+
+define run_engine_unit_test
+	@echo "running unit test for packages:" $(1)
+	mkdir -p $(ENGINE_TEST_DIR)
+	$(FAILPOINT_ENABLE)
+	@export log_level=error; \
+	$(GOTEST) -timeout 5m -covermode=atomic -coverprofile="$(ENGINE_TEST_DIR)/cov.unit_test.out" $(1) \
+	|| { $(FAILPOINT_DISABLE); exit 1; }
+	$(FAILPOINT_DISABLE)
+endef

engine/chaos/cases/cases.go

@@ -11,6 +11,19 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package main
 
 import (

engine/chaos/cases/config.go

@@ -1,3 +1,16 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 // Copyright 2020 PingCAP, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");