Functional Tests (with Non-Cloud Resources)

Functional Tests (with Non-Cloud Resources) #1419

Workflow file for this run

.github/workflows/functional-test-noncloud.yaml at 2bc71ab

	# ------------------------------------------------------------
	# Copyright 2023 The Radius Authors.
	#
	# Licensed under the Apache License, Version 2.0 (the "License");
	# you may not use this file except in compliance with the License.
	# You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.
	# ------------------------------------------------------------

	name: Functional Tests (with Non-Cloud Resources)

	permissions:
	id-token: write # Required for requesting the JWT
	contents: read # Required for listing the commits
	packages: write # Required for uploading the package
	checks: write # Required for creating a check run

	on:
	# Enable manual trigger
	workflow_dispatch:
	inputs:
	branch:
	description: "Branch to run the workflow on"
	required: true
	default: "main"
	schedule:
	# Run every 4 hours on weekdays.
	- cron: "30 0,4,8,12,16,20 * * 1-5"
	# Run every 12 hours on weekends.
	- cron: "30 0,12 * * 0,6"
	# Dispatch on external events
	repository_dispatch:
	types: [de-functional-test]
	pull_request:
	branches:
	- main
	- features/*
	- release/*

	env:
	# Go version
	GOVER: "1.22.5"
	# Helm version
	HELM_VER: "v3.15.3"
	# KinD cluster version
	KIND_VER: "v0.23.0"
	# Dapr version
	DAPR_VER: "1.12.0"
	# Dapr dashboard version
	DAPR_DASHBOARD_VER: "0.14.0"
	# Kubectl version
	KUBECTL_VER: "v1.30.0"

	# The radius functional test timeout
	FUNCTIONALTEST_TIMEOUT: 15m
	# The base directory for storing test logs
	RADIUS_CONTAINER_LOG_BASE: dist/container_logs
	# The Radius helm chart location.
	RADIUS_CHART_LOCATION: deploy/Chart/
	# The current GitHub action link
	ACTION_LINK: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
	# Server where terraform test modules are deployed
	TF_RECIPE_MODULE_SERVER_URL: "http://tf-module-server.radius-test-tf-module-server.svc.cluster.local"
	# Private Git repository where terraform module for testing is stored.
	TF_RECIPE_PRIVATE_GIT_SOURCE: "git::https://github.com/radius-project/terraform-private-modules//kubernetes-redis"

	# The number of failed tests to report.
	ISSUE_CREATE_THRESHOLD: 2

	# Local Docker registry name
	LOCAL_REGISTRY_NAME: "radius-registry"
	# Local Docker registry server
	LOCAL_REGISTRY_SERVER: "localhost"
	# Local Docker registry port
	LOCAL_REGISTRY_PORT: "5000"
	# bicep-types ACR url for uploading Radius Bicep types
	BICEP_TYPES_REGISTRY: 'biceptypes.azurecr.io'

	jobs:
	build:
	name: Build Radius for test
	runs-on: ubuntu-latest
	env:
	DE_IMAGE: "ghcr.io/radius-project/deployment-engine"
	DE_TAG: "latest"
	outputs:
	REL_VERSION: ${{ steps.gen-id.outputs.REL_VERSION }}
	DE_IMAGE: ${{ steps.gen-id.outputs.DE_IMAGE }}
	DE_TAG: ${{ steps.gen-id.outputs.DE_TAG }}
	steps:
	- name: Set DE image and tag (repository_dispatch from de-functional-test)
	if: github.event_name == 'repository_dispatch'
	uses: actions/github-script@v7
	with:
	github-token: ${{ secrets.GH_RAD_CI_BOT_PAT }}
	script: \|
	const clientPayload = context.payload.client_payload;
	if (clientPayload && clientPayload.event_type === `de-functional-test`) {
	var fs = require('fs');
	// Set environment variables
	fs.appendFileSync(process.env.GITHUB_ENV,
	`DE_IMAGE=${clientPayload.de_image}\n`+
	`DE_TAG=${clientPayload.de_tag}\n`+
	`CHECKOUT_REPO=${{ github.repository }}\n`+
	`CHECKOUT_REF=refs/heads/main`
	);
	}

	- name: Generate ID for release
	id: gen-id
	run: \|
	BASE_STR="RADIUS\|${GITHUB_SHA}\|${GITHUB_SERVER_URL}\|${GITHUB_REPOSITORY}\|${GITHUB_RUN_ID}\|${GITHUB_RUN_ATTEMPT}"
	if [ "$GITHUB_EVENT_NAME" == "schedule" ]; then
	# Add run number to randomize unique id for scheduled runs.
	BASE_STR="${GITHUB_RUN_NUMBER}\|${BASE_STR}"
	fi
	UNIQUE_ID=func$(echo $BASE_STR \| sha1sum \| head -c 10)
	echo "REL_VERSION=pr-${UNIQUE_ID}" >> $GITHUB_ENV

	# Set output variables to be used in the other jobs
	echo "REL_VERSION=pr-${UNIQUE_ID}" >> $GITHUB_OUTPUT
	echo "DE_IMAGE=${{ env.DE_IMAGE }}" >> $GITHUB_OUTPUT
	echo "DE_TAG=${{ env.DE_TAG }}" >> $GITHUB_OUTPUT
	tests:
	name: Run ${{ matrix.name }} functional tests
	needs: build
	strategy:
	fail-fast: true
	matrix:
	os: [ubuntu-latest-m]
	name:
	[
	cli-noncloud,
	corerp-noncloud,
	daprrp-noncloud,
	kubernetes-noncloud,
	msgrp-noncloud,
	samples-noncloud,
	ucp-noncloud,
	datastoresrp-noncloud,
	]
	runs-on: ${{ matrix.os }}
	env:
	REL_VERSION: ${{ needs.build.outputs.REL_VERSION }}
	BICEP_RECIPE_TAG_VERSION: ${{ needs.build.outputs.REL_VERSION }}
	DE_IMAGE: ${{ needs.build.outputs.DE_IMAGE }}
	DE_TAG: ${{ needs.build.outputs.DE_TAG }}
	steps:
	- name: Set up checkout target (scheduled)
	if: github.event_name == 'schedule' \|\| github.event_name == 'repository_dispatch'
	run: \|
	echo "CHECKOUT_REPO=${{ github.repository }}" >> $GITHUB_ENV
	echo "CHECKOUT_REF=refs/heads/main" >> $GITHUB_ENV

	- name: Set up checkout target (pull_request)
	if: github.event_name == 'pull_request'
	run: \|
	echo "CHECKOUT_REPO=${{ github.repository }}" >> $GITHUB_ENV
	echo "CHECKOUT_REF=${{ github.ref }}" >> $GITHUB_ENV
	echo "PR_NUMBER=${{ github.event.pull_request.number }}" >> $GITHUB_ENV

	- name: Set up checkout target (workflow_dispatch)
	if: github.event_name == 'workflow_dispatch'
	run: \|
	echo "CHECKOUT_REPO=${{ github.repository }}" >> $GITHUB_ENV
	echo "CHECKOUT_REF=refs/heads/${{ github.event.inputs.branch }}" >> $GITHUB_ENV

	- name: Check out code
	uses: actions/checkout@v4

	- name: Checkout samples repo
	uses: actions/checkout@v4
	if: matrix.name == 'samples-noncloud'
	with:
	repository: radius-project/samples
	ref: refs/heads/edge
	path: samples

	- name: Set up Go ${{ env.GOVER }}
	uses: actions/setup-go@v5
	with:
	go-version: ${{ env.GOVER }}

	- name: Get Go Cache path
	id: go-cache-paths
	run: \|
	echo "go-build=$(go env GOCACHE)" >> $GITHUB_OUTPUT
	sudo rm -rf $(go env GOCACHE)

	echo "go-mod=$(go env GOMODCACHE)" >> $GITHUB_OUTPUT
	sudo rm -rf $(go env GOMODCACHE)

	- uses: actions/cache@v4
	with:
	path: \|
	${{ steps.go-cache-paths.outputs.go-build }}
	${{ steps.go-cache-paths.outputs.go-mod }}
	key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
	restore-keys: \|
	${{ runner.os }}-go-

	- name: Setup Node.js
	uses: actions/setup-node@v4
	with:
	node-version: '18'

	- name: Generate Bicep extensibility types from OpenAPI specs
	run: \|
	make generate-bicep-types VERSION=${{ env.REL_VERSION == 'edge' && 'latest' \|\| env.REL_VERSION }}

	- name: Upload Radius Bicep types artifacts
	uses: actions/upload-artifact@v4
	with:
	name: ${{ matrix.name }}_radius_bicep_types
	path: ./hack/bicep-types-radius/generated
	if-no-files-found: error

	- name: Create a secure local registry
	id: create-local-registry
	uses: ./.github/actions/create-local-registry
	with:
	secure: "true"
	registry-name: ${{ env.LOCAL_REGISTRY_NAME }}
	registry-server: ${{ env.LOCAL_REGISTRY_SERVER }}
	registry-port: ${{ env.LOCAL_REGISTRY_PORT }}

	- name: Build and Push container images
	run: \|
	make build && make docker-build && make docker-push
	env:
	DOCKER_REGISTRY: "${{ env.LOCAL_REGISTRY_SERVER }}:${{ env.LOCAL_REGISTRY_PORT }}"
	DOCKER_TAG_VERSION: ${{ env.REL_VERSION }}

	- name: Install rad CLI
	run: \|
	mkdir ./bin
	cp ./dist/linux_amd64/release/rad ./bin/rad
	chmod +x ./bin/rad
	export PATH=$GITHUB_WORKSPACE/bin:$PATH
	which rad \|\| { echo "cannot find rad"; exit 1; }
	rad bicep download
	rad version

	- uses: azure/setup-helm@v4
	with:
	version: ${{ env.HELM_VER }}

	- name: Create a KinD cluster with a local registry
	uses: ./.github/actions/create-kind-cluster
	with:
	secure: "true"
	temp-cert-dir: ${{ steps.create-local-registry.outputs.temp-cert-dir }}
	kind-version: ${{ env.KIND_VER }}
	with-local-registry: "true"
	registry-name: ${{ env.LOCAL_REGISTRY_NAME }}
	registry-server: ${{ env.LOCAL_REGISTRY_SERVER }}
	registry-port: ${{ env.LOCAL_REGISTRY_PORT }}

	- name: Install dapr into cluster
	run: \|
	wget -q https://raw.githubusercontent.com/dapr/cli/master/install/install.sh -O - \| /bin/bash -s ${{ env.DAPR_VER }}
	dapr init -k --wait --timeout 600 --runtime-version ${{ env.DAPR_VER }} --dashboard-version ${{ env.DAPR_DASHBOARD_VER }}

	- name: Install Radius
	run: \|
	export PATH=$GITHUB_WORKSPACE/bin:$PATH
	which rad \|\| { echo "cannot find rad"; exit 1; }

	# Check if Radius release exists and delete it if found
	if helm status radius -n radius-system >/dev/null 2>&1; then
	echo "The release 'radius' exists. Deleting the release..."
	helm delete radius -n radius-system
	if [ $? -eq 0 ]; then
	echo "Release 'radius' deleted successfully."
	else
	echo "Failed to delete the release 'radius'."
	exit 1
	fi
	else
	echo "Radius release not found. Proceeding with installation."
	fi

	RAD_COMMAND="rad install kubernetes \
	--chart ${{ env.RADIUS_CHART_LOCATION }} \
	--set rp.image=${{ env.LOCAL_REGISTRY_NAME }}:${{ env.LOCAL_REGISTRY_PORT }}/applications-rp,rp.tag=${{ env.REL_VERSION }} \
	--set controller.image=${{ env.LOCAL_REGISTRY_NAME }}:${{ env.LOCAL_REGISTRY_PORT }}/controller,controller.tag=${{ env.REL_VERSION }} \
	--set ucp.image=${{ env.LOCAL_REGISTRY_NAME }}:${{ env.LOCAL_REGISTRY_PORT }}/ucpd,ucp.tag=${{ env.REL_VERSION }} \
	--set de.image=${{ env.DE_IMAGE }},de.tag=${{ env.DE_TAG }}"

	if [ "${{ env.USE_CERT_FILE }}" = "true" ]; then
	RAD_COMMAND="$RAD_COMMAND --set-file global.rootCA.cert=$TEMP_CERT_DIR/certs/${{ env.LOCAL_REGISTRY_SERVER }}/client.crt"
	fi

	echo "* Installing Radius to Kubernetes *"
	eval $RAD_COMMAND

	echo "* Create workspace, group and environment for test *"
	rad workspace create kubernetes
	rad group create kind-radius
	rad group switch kind-radius

	# The functional test is designed to use default namespace. So you must create the environment for default namespace.
	rad env create kind-radius --namespace default
	rad env switch kind-radius
	env:
	USE_CERT_FILE: "true"
	TEMP_CERT_DIR: ${{ steps.create-local-registry.outputs.temp-cert-dir }}

	- name: Publish Terraform test recipes
	run: \|
	make publish-test-terraform-recipes

	- name: Setup and verify bicep CLI
	run: \|
	curl -Lo bicep https://github.com/Azure/bicep/releases/latest/download/bicep-linux-x64
	chmod +x ./bicep
	sudo mv ./bicep /usr/local/bin/bicep
	bicep --version

	- name: Publish bicep types
	run: \|
	bicep publish-extension ./hack/bicep-types-radius/generated/index.json --target br:${{ env.LOCAL_REGISTRY_SERVER }}:${{ env.LOCAL_REGISTRY_PORT }}/radius:${{ env.REL_VERSION == 'edge' && 'latest' \|\| env.REL_VERSION }} --force

	- name: Generate test bicepconfig.json
	run: \|
	if [[ "${{ env.REL_VERSION }}" == "edge" ]]; then
	RADIUS_VERSION="latest"
	else
	RADIUS_VERSION="${{ env.REL_VERSION }}"
	fi
	cat <<EOF > ./test/bicepconfig.json
	{
	"experimentalFeaturesEnabled": {
	"extensibility": true
	},
	"extensions": {
	"radius": "br:${{ env.LOCAL_REGISTRY_SERVER }}:${{ env.LOCAL_REGISTRY_PORT }}/radius:$RADIUS_VERSION",
	"aws": "br:${{ env.BICEP_TYPES_REGISTRY }}/aws:latest"
	}
	}
	EOF

	- name: Publish Bicep Test Recipes
	run: \|
	export PATH=$GITHUB_WORKSPACE/bin:$PATH
	which rad \|\| { echo "cannot find rad"; exit 1; }
	make publish-test-bicep-recipes
	env:
	BICEP_RECIPE_REGISTRY: "${{ env.LOCAL_REGISTRY_SERVER }}:${{ env.LOCAL_REGISTRY_PORT }}"
	BICEP_RECIPE_TAG_VERSION: ${{ env.REL_VERSION }}
	TEMP_CERT_DIR: ${{ steps.create-local-registry.outputs.temp-cert-dir }}
	SSL_CERT_FILE: ${{ steps.create-local-registry.outputs.temp-cert-dir }}/certs/${{ env.LOCAL_REGISTRY_SERVER }}/client.crt

	- name: Run functional tests
	run: \|
	# Ensure rad cli is in path before running tests.
	export PATH=$GITHUB_WORKSPACE/bin:$PATH
	# Make directory to capture functional test results
	mkdir -p ./dist/functional_test
	cd $GITHUB_WORKSPACE

	which rad \|\| { echo "cannot find rad"; exit 1; }

	make test-functional-${{ matrix.name }}
	env:
	DOCKER_REGISTRY: "${{ env.LOCAL_REGISTRY_NAME }}:${{ env.LOCAL_REGISTRY_PORT }}"
	TEST_TIMEOUT: ${{ env.FUNCTIONALTEST_TIMEOUT }}
	RADIUS_CONTAINER_LOG_PATH: ${{ github.workspace }}/${{ env.RADIUS_CONTAINER_LOG_BASE }}
	RADIUS_SAMPLES_REPO_ROOT: ${{ github.workspace }}/samples
	BICEP_RECIPE_REGISTRY: "${{ env.LOCAL_REGISTRY_NAME }}:${{ env.LOCAL_REGISTRY_PORT }}"
	BICEP_RECIPE_TAG_VERSION: ${{ env.BICEP_RECIPE_TAG_VERSION }}
	GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	GOTESTSUM_OPTS: "--junitfile ./dist/functional_test/results.xml"

	- name: Process Functional Test Results
	uses: ./.github/actions/process-test-results
	# In case of failure, upload functional_test_results to artifacts so that they are not erased by subsequent runs.
	if: failure() && github.repository == 'radius-project/radius'
	with:
	test_group_name: "Functional Tests - ${{ matrix.name }}"
	artifact_name: "functional_test_results_${{ matrix.name }}"
	result_directory: "dist/functional_test/"

	- name: Collect detailed Radius logs and events
	id: radius-logs-events
	if: always()
	run: \|
	# Create Radius logs directory
	mkdir -p func-nc/radius-logs-events/${{ matrix.name }}

	# Get pod logs and save to file
	namespace="radius-system"
	pod_names=($(kubectl get pods -n $namespace -o jsonpath='{.items[*].metadata.name}'))
	for pod_name in "${pod_names[@]}"; do
	kubectl logs $pod_name -n $namespace > func-nc/radius-logs-events/${{ matrix.name }}/${pod_name}.txt
	done
	echo "Pod logs saved to func-nc/radius-logs-events/${{ matrix.name }}/"
	# Get kubernetes events and save to file
	kubectl get events -n $namespace > func-nc/radius-logs-events/${{ matrix.name }}/events.txt

	- name: Upload Pod logs for failed tests
	uses: actions/upload-artifact@v4
	if: always() && steps.radius-logs-events.outcome == 'success'
	with:
	name: ${{ matrix.name }}-radius-pod-logs
	path: func-nc/radius-logs-events/${{ matrix.name }}
	retention-days: 30
	if-no-files-found: error

	- name: Collect Pod details
	if: always()
	run: \|
	POD_STATE_LOG_FILENAME='${{ env.RADIUS_CONTAINER_LOG_BASE }}/${{ matrix.name }}-tests-pod-states.log'
	mkdir -p $(dirname $POD_STATE_LOG_FILENAME)
	echo "kubectl get pods -A" >> $POD_STATE_LOG_FILENAME
	kubectl get pods -A >> $POD_STATE_LOG_FILENAME
	echo "kubectl describe pods -A" >> $POD_STATE_LOG_FILENAME
	kubectl describe pods -A >> $POD_STATE_LOG_FILENAME

	- name: Upload container logs
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: ${{ matrix.name }}_container_logs
	path: ./${{ env.RADIUS_CONTAINER_LOG_BASE }}

	- name: Get Terraform recipe publishing logs
	if: always()
	run: \|
	# Create pod-logs directory
	mkdir -p recipes/pod-logs
	# Get pod logs and save to file
	namespace="radius-test-tf-module-server"
	label="app.kubernetes.io/name=tf-module-server"
	pod_names=($(kubectl get pods -l $label -n $namespace -o jsonpath='{.items[*].metadata.name}'))
	for pod_name in "${pod_names[@]}"; do
	kubectl logs $pod_name -n $namespace > recipes/pod-logs/${pod_name}.txt
	done
	echo "Pod logs saved to recipes/pod-logs/"
	# Get kubernetes events and save to file
	kubectl get events -n $namespace > recipes/pod-logs/events.txt

	- name: Upload Terraform recipe publishing logs
	uses: actions/upload-artifact@v4
	if: always()
	with:
	name: ${{ matrix.name }}_recipes-pod-logs
	path: recipes/pod-logs
	if-no-files-found: error

	report-failure:
	name: Report test failure
	needs: [build, tests]
	runs-on: ubuntu-latest
	if: failure() && github.event_name == 'schedule' && github.repository == 'radius-project/radius'
	steps:
	- name: Count recently failed tests
	id: count_failures
	uses: actions/github-script@v7
	with:
	script: \|
	response = await github.rest.actions.listWorkflowRuns({
	owner: context.repo.owner,
	repo: context.repo.repo,
	workflow_id: 'functional-test-noncloud.yaml',
	event: 'schedule',
	per_page: 10
	});

	failureCount = 1;
	for (const run of response.data.workflow_runs) {
	if (run.conclusion === 'failure') {
	failureCount++;
	} else {
	break;
	}
	}
	return failureCount;
	- name: Create failure issue for failing scheduled run
	uses: actions/github-script@v7
	# Only create an issue if there are (env.ISSUE_CREATE_THRESHOLD) failures of the recent tests.
	if: steps.count_failures.outputs.result >= env.ISSUE_CREATE_THRESHOLD
	with:
	github-token: ${{ secrets.GH_RAD_CI_BOT_PAT }}
	script: \|
	github.rest.issues.create({
	...context.repo,
	title: `Scheduled functional test (noncloud) failed - Run ID: ${context.runId}`,
	labels: ['bug', 'test-failure'],
	body: `## Bug information \n\nThis bug is generated automatically if the scheduled functional test fails at least ${process.env.ISSUE_CREATE_THRESHOLD} times in a row. The Radius functional test operates on a schedule of every 4 hours during weekdays and every 12 hours over the weekend. It's important to understand that the test may fail due to workflow infrastructure issues, like network problems, rather than the flakiness of the test itself. For the further investigation, please visit [here](${process.env.ACTION_LINK}).`
	})

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Functional Tests (with Non-Cloud Resources) #1419

Workflow file

Functional Tests (with Non-Cloud Resources) #1419

Jobs

Run details

Workflow file for this run