Skip to content

Commit

Permalink
test: implement new environment setup
Browse files Browse the repository at this point in the history
Co-authored-by: Jonas Lammler <jonas.lammler@hetzner-cloud.de>
Co-authored-by: Julian Tölle <julian.toelle@hetzner-cloud.de>
  • Loading branch information
3 people committed Jun 21, 2024
1 parent 9a4f97d commit 26c57af
Show file tree
Hide file tree
Showing 23 changed files with 533 additions and 306 deletions.
78 changes: 43 additions & 35 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@ jobs:
with:
token: ${{ secrets.CODECOV_TOKEN }}

test-kubernetes:
name: e2e k3s ${{ matrix.k3s }}
kubernetes:
name: kubernetes ${{ matrix.k3s }}
runs-on: ubuntu-latest

permissions:
Expand All @@ -33,21 +33,22 @@ jobs:
concurrency: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.k3s }}

strategy:
# The e2e tests are flaky and often one of the jobs fails. The default setting
# causes all other currently running jobs to abort and all need to be restarted.
fail-fast: false
fail-fast: false # Continue tests matrix if a flaky run occur.
matrix:
include:
# All k3s after January 2024 break our e2e tests, we hardcode
# the versions for now until we can fix the source of this.
- k3s: v1.26.12+k3s1
k8s-test: v1.26.12
- k3s: v1.27.9+k3s1
k8s-test: v1.27.9
- k3s: v1.28.5+k3s1
k8s-test: v1.28.5
- k3s: v1.29.0+k3s1
k8s-test: v1.29.0
- k3s: v1.26
k8s-test: v1.26.15
- k3s: v1.27
k8s-test: v1.27.15
- k3s: v1.28
k8s-test: v1.28.11
- k3s: v1.29
k8s-test: v1.29.6

env:
K3S_CHANNEL: ${{ matrix.k3s }}
K8S_TEST_VERSION: ${{ matrix.k8s-test }}
ENV: gha-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.k3s }}

steps:
- uses: actions/checkout@v4
Expand All @@ -56,36 +57,43 @@ jobs:
with:
go-version-file: go.mod

- uses: hetznercloud/tps-action@main
- uses: opentofu/setup-opentofu@v1
with:
tofu_version: v1.7.2
tofu_wrapper: false

- uses: hetznercloud/setup-hcloud@v1
- uses: docker/setup-buildx-action@v3

- uses: yokawasa/action-setup-kube-tools@v0.11.1
with:
setup-tools: |
helm
kubectl
skaffold
helm: v3.11.2
kubectl: v1.29.0
skaffold: v2.3.0
helm: v3.15.2 # renovate: datasource=github-releases depName=helm/helm
kubectl: v1.29.6 # renovate: datasource=github-releases depName=kubernetes/kubernetes
skaffold: v2.12.0 # renovate: datasource=github-releases depName=GoogleContainerTools/skaffold

- name: Run tests
env:
K3S_VERSION: ${{ matrix.k3s }}
K8S_TEST_VERSION: ${{ matrix.k8s-test }}
SCOPE: gha-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.k3s }}
run: |
curl -sLS https://get.k3sup.dev | sh
- name: Setup k3sup
run: curl -sLS https://get.k3sup.dev | sh

- name: Setup k8s test binaries
run: make -C test/e2e/kubernetes bin

trap "hack/dev-down.sh" EXIT
source <(hack/dev-up.sh)
- uses: hetznercloud/tps-action@main

- name: Setup environment
run: make -C dev up

skaffold build --tag="e2e-${GITHUB_RUN_ID}-${GITHUB_RUN_NUMBER}"
tag=$(skaffold build --tag="e2e-${GITHUB_RUN_ID}-${GITHUB_RUN_NUMBER}" --quiet --output="{{ (index .Builds 0).Tag }}")
skaffold deploy --images=docker.io/hetznercloud/hcloud-csi-driver=$tag
- name: Run skaffold
run: |
source dev/files/env.sh
skaffold run
test/e2e/kubernetes/run-e2e-tests.sh
- name: Run tests
run: |
source dev/files/env.sh
make -C test/e2e/kubernetes parallel serial
deploy-manifests:
runs-on: ubuntu-latest
Expand All @@ -95,7 +103,7 @@ jobs:
with:
setup-tools: |
helm
helm: v3.11.2
helm: v3.11.2 # renovate: datasource=github-releases depName=helm/helm

- uses: actions/checkout@v4

Expand All @@ -117,7 +125,7 @@ jobs:
with:
setup-tools: |
helm
helm: v3.11.2
helm: v3.11.2 # renovate: datasource=github-releases depName=helm/helm

- uses: actions/checkout@v4

Expand Down
3 changes: 0 additions & 3 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,9 +1,6 @@
# rootfs for docker plugin
deploy/docker-swarm/pkg/plugin

# Tmp files for dev env
hack/.*

# Build output
dist/
hcloud-csi-*.tgz
Expand Down
9 changes: 9 additions & 0 deletions dev/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
.terraform*
terraform.tfstate
terraform.tfstate.backup
*.auto.tfvars

files/*
!files/.gitkeep

.env
File renamed without changes.
27 changes: 27 additions & 0 deletions dev/Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
SHELL = bash
.ONESHELL:

ENV ?= dev
K3S_CHANNEL ?= stable

env.auto.tfvars:
@echo 'name = "$(ENV)"' > "$@"
@echo 'hcloud_token = "$(HCLOUD_TOKEN)"' >> "$@"
@echo 'k3s_channel = "$(K3S_CHANNEL)"' >> "$@"

.terraform:
tofu init

up: .terraform env.auto.tfvars
tofu apply -auto-approve
$(MAKE) port-forward

down: .terraform env.auto.tfvars
tofu destroy -auto-approve

port-forward:
source files/env.sh
bash files/registry-port-forward.sh

clean:
rm -Rf files/* .terraform* terraform.tfstate* env.auto.tfvars
Empty file added dev/files/.gitkeep
Empty file.
102 changes: 102 additions & 0 deletions dev/hcloud-k8s-env/main-infra.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
# Setup the infrastructure

provider "hcloud" {
token = var.hcloud_token
}

locals {
labels = {
env = var.name
}
}

# SSH Key

resource "tls_private_key" "ssh" {
algorithm = "ED25519"
}

resource "local_sensitive_file" "ssh" {
content = tls_private_key.ssh.private_key_openssh
filename = abspath("${path.root}/files/id_ed25519")
}

resource "hcloud_ssh_key" "default" {
name = var.name
public_key = tls_private_key.ssh.public_key_openssh
labels = local.labels
}

# Network

resource "hcloud_network" "cluster" {
name = var.name
ip_range = "10.0.0.0/8"
labels = local.labels
}

resource "hcloud_network_subnet" "cluster" {
network_id = hcloud_network.cluster.id
network_zone = "eu-central"
type = "cloud"
ip_range = "10.0.0.0/24"
}

# Control Plane Node

resource "hcloud_server" "control" {
name = "${var.name}-control"
server_type = var.hcloud_server_type
location = var.hcloud_location
image = var.hcloud_image
ssh_keys = [hcloud_ssh_key.default.id]
labels = local.labels

connection {
host = self.ipv4_address
private_key = tls_private_key.ssh.private_key_openssh
}

provisioner "remote-exec" {
inline = ["cloud-init status --wait || test $? -eq 2"]
}
}

resource "hcloud_server_network" "control" {
server_id = hcloud_server.control.id
subnet_id = hcloud_network_subnet.cluster.id
}

# Worker / Agent Nodes

variable "worker_count" {
type = number
default = 3
}

resource "hcloud_server" "worker" {
count = var.worker_count

name = "${var.name}-worker-${count.index}"
server_type = var.hcloud_server_type
location = var.hcloud_location
image = var.hcloud_image
ssh_keys = [hcloud_ssh_key.default.id]
labels = local.labels

connection {
host = self.ipv4_address
private_key = tls_private_key.ssh.private_key_openssh
}

provisioner "remote-exec" {
inline = ["cloud-init status --wait || test $? -eq 2"]
}
}

resource "hcloud_server_network" "worker" {
count = var.worker_count

server_id = hcloud_server.worker[count.index].id
subnet_id = hcloud_network_subnet.cluster.id
}
Loading

0 comments on commit 26c57af

Please sign in to comment.