Merge branch 'main' into sts/proto_stream_node_append_only

.github/workflow-template/generate.sh

@@ -42,6 +42,23 @@ echo "$HEADER" > ../workflows/main.yml
 yq ea '. as $item ireduce ({}; . * $item )' template.yml main-override.yml "${jobs_main[@]}" | yq eval '... comments=""' - >> ../workflows/main.yml
 echo "$HEADER" >> ../workflows/main.yml
 
+# Generate workflow for main cronjob
+# TODO add connector specific jobs below
+
+jobs_main_cron=(
+   "jobs/frontend-check.yml"
+   "jobs/e2e-risedev-dev.gen.yml"
+   "jobs/e2e-source.yml"
+   "jobs/compute-node-build-dev.gen.yml"
+   "jobs/compute-node-test.yml"
+   "jobs/misc-check.yml"
+)
+
+echo "$HEADER" > ../workflows/main-cron.yml
+# shellcheck disable=SC2016
+yq ea '. as $item ireduce ({}; . * $item )' template.yml main-cron.yml "${jobs_main_cron[@]}" | yq eval '... comments=""' - >> ../workflows/main-cron.yml
+echo "$HEADER" >> ../workflows/main-cron.yml
+
 # Generate workflow for pull requests
 
 jobs_pr=(

.github/workflow-template/main-cron.yml

@@ -0,0 +1,60 @@
+# Override `template.yml` with contents in this file. Please run `generate.sh` to update the corresponding files.
+
+name: CI (main-cronjob)
+
+on:
+  # Triggers the workflow everyday 00:00
+  schedule:
+    - cron: '0 0 * * *'
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  start-runner-c:
+    name: ec2-start-c
+    runs-on: ubuntu-latest
+    outputs:
+      label: ${{ steps.start-ec2-runner.outputs.label }}
+      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
+    steps:
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-2
+      - name: Start EC2 runner
+        id: start-ec2-runner
+        uses: machulav/ec2-github-runner@v2
+        with:
+          mode: start
+          github-token: ${{ secrets.WUTAO_PAT }}
+          ec2-image-id: ami-05a99cf0eb100c377 # github-actions (60gb gp3 ssd)
+          ec2-instance-type: c5.4xlarge # 16 vCPU, 32 GB Memory, EBS
+          subnet-id: subnet-59743123 # default
+          security-group-id: sg-9ec32bfe # default (could have security issues)
+
+  stop-runner-c:
+    name: ec2-stop-c
+    needs:
+      - start-runner-c
+      - end-to-end-risedev-release
+    runs-on: ubuntu-latest
+    if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-2
+      - name: Stop EC2 runner C
+        uses: machulav/ec2-github-runner@v2
+        with:
+          mode: stop
+          github-token: ${{ secrets.WUTAO_PAT }}
+          label: ${{ needs.start-runner-c.outputs.label }}
+          ec2-instance-id: ${{ needs.start-runner-c.outputs.ec2-instance-id }}
+        if: ${{ always() }}
+