-
Notifications
You must be signed in to change notification settings - Fork 500
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
update eks e2e script and jenkins file
- Loading branch information
Showing
9 changed files
with
353 additions
and
38 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,179 @@ | ||
| #!/bin/bash | ||
|
|
||
| # Copyright 2020 PingCAP, Inc. | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| # | ||
| # aws-k8s-tester cannot clean all resources created when some error happened. | ||
| # This script is used to clean resources created by aws-k8s-tester in our CI. | ||
| # | ||
| # DO NOT USE THIS SCRIPT FOR OTHER USES! | ||
| # | ||
|
|
||
| function iam_role_exists() { | ||
| local roleName="$1" | ||
| for n in $(aws iam list-roles --no-paginate --query 'Roles[*].RoleName' --output text); do | ||
| if [[ "$roleName" == "$n" ]]; then | ||
| return 0 | ||
| fi | ||
| done | ||
| return 1 | ||
| } | ||
|
|
||
| function delete_role() { | ||
| local roleName="$1" | ||
| if ! iam_role_exists "$roleName"; then | ||
| echo "info: IAM role '$roleName' does not exist, skipped" | ||
| return | ||
| fi | ||
| # detach role policies first | ||
| for policyArn in $(aws iam list-attached-role-policies --role-name "$roleName" --query 'AttachedPolicies[*].PolicyArn' --output text); do | ||
| aws iam detach-role-policy --role-name "$roleName" --policy-arn "$policyArn" | ||
| done | ||
| aws iam delete-role --role-name "$roleName" | ||
| } | ||
|
|
||
| # delete-vpc cannot delete its dependencies, so we need to remove one by one | ||
| # https://github.com/aws/aws-cli/issues/1721 | ||
| # https://aws.amazon.com/premiumsupport/knowledge-center/troubleshoot-dependency-error-delete-vpc/ | ||
| function delete_vpc() { | ||
| local vpc="$1" | ||
| echo "info: checking associated instances" | ||
| for id in $(aws ec2 describe-instances --filters 'Name=vpc-id,Values='$vpc --query 'Reservations[*].Instances[*].InstanceId' --output text); do | ||
| echo aws ec2 delete-instance --group-id $id | ||
| done | ||
| echo "info: checking associated security groups" | ||
| while IFS=$'\n' read -r line; do | ||
| read -r id name <<< $line | ||
| if [[ "$name" != "default" ]]; then | ||
| aws ec2 delete-security-group --group-id $id | ||
| fi | ||
| done <<< $(aws ec2 describe-security-groups --filters 'Name=vpc-id,Values='$vpc --query 'SecurityGroups[*].{Name:GroupName,Id:GroupId}' --output text) | ||
| echo "info: checking associated internet gateways" | ||
| for id in $(aws ec2 describe-internet-gateways --filters 'Name=attachment.vpc-id,Values='$vpc --query 'InternetGateways[*].InternetGatewayId' --output text); do | ||
| aws ec2 delete-internet-gateway --internet-gateway-id $id | ||
| done | ||
| echo "info: checking associated subnets" | ||
| for id in $(aws ec2 describe-subnets --filters 'Name=vpc-id,Values='$vpc --query 'Subnets[*].SubnetId' --output text); do | ||
| aws ec2 delete-subnet --subnet-id $id | ||
| done | ||
| echo "info: checking associated route tables" | ||
| for id in $(aws ec2 describe-route-tables --filters 'Name=vpc-id,Values='$vpc --query 'RouteTables[*].RouteTableId' --output text); do | ||
| aws ec2 delete-route-table --route-table-id $id | ||
| done | ||
| echo "info: checking associated network acls" | ||
| for id in $(aws ec2 describe-network-acls --filters 'Name=vpc-id,Values='$vpc --query 'NetworkAcls[*].NetworkAclId' --output text); do | ||
| aws ec2 delete-network-acl --network-acl-id $id | ||
| done | ||
| echo "info: checking associated vpc endpoints" | ||
| for id in $(aws ec2 describe-vpc-endpoints --filters 'Name=vpc-id,Values='$vpc --query 'VpcEndpoints[*].VpcEndpointId' --output text); do | ||
| aws ec2 delete-vpc-endpoint --vpc-endpoint-id $id | ||
| done | ||
| echo "info: checking associated nat gateways" | ||
| for id in $(aws ec2 describe-nat-gateways --filter 'Name=vpc-id,Values='$vpc --query 'NatGateways[*].NatGatewayId' --output text); do | ||
| aws ec2 delete-nat-gateway --nat-gateway-id $id | ||
| done | ||
| # echo "info: checking associated vpn connections" | ||
| # for id in $(aws ec2 describe-vpn-connections --filters 'Name=vpc-id,Values='$vpc --query 'VpcConnections[*].VpcConnectionId' --output text); do | ||
| # aws ec2 delete-vpn-connection --vpn-connection-id $id | ||
| # done | ||
| # echo "info: checking associated vpn gateways" | ||
| # for id in $(aws ec2 describe-vpn-gateways --filters 'Name=vpc-id,Values='$vpc --query 'VpcConnections[*].VpcConnectionId' --output text); do | ||
| # aws ec2 delete-vpn-gateway --vpn-gateway-id $id | ||
| # done | ||
| echo "info: checking associated network interfaces" | ||
| for id in $(aws ec2 describe-network-interfaces --filters 'Name=vpc-id,Values='$vpc --query 'NetworkInterfaces[*].NetworkInterfaceId' --output text); do | ||
| aws ec2 delete-network-interface --network-interface-id $id | ||
| done | ||
| } | ||
|
|
||
| function get_stacks() { | ||
| aws cloudformation list-stacks --stack-status-filter CREATE_COMPLETE DELETE_FAILED --query 'StackSummaries[*].StackName' --output text | ||
| } | ||
|
|
||
| function delete_ec2_key_pair() { | ||
| local name="$1" | ||
| local n=$(aws ec2 describe-key-pairs --key-names $name --output text 2>/dev/null | wc -l) | ||
| if [ "$n" -gt 0 ]; then | ||
| aws ec2 delete-key-pair --key-name $name | ||
| fi | ||
| } | ||
|
|
||
| function clean_eks() { | ||
| echo "info: deleting mng stack" | ||
| local regex='^'$CLUSTER'-mng-[0-9]+$' | ||
| local mngStack= | ||
| for stackName in $(get_stacks); do | ||
| if [[ ! "$stackName" =~ $regex ]]; then | ||
| continue | ||
| fi | ||
| mngStack=$stackName | ||
| break | ||
| done | ||
| if [ -n "$mngStack" ]; then | ||
| echo "info: mng stack found '$mngStack', deleting it" | ||
| aws cloudformation delete-stack --stack-name $mngStack | ||
| aws cloudformation wait stack-delete-complete --stack-name $mngStack | ||
| else | ||
| echo "info: mng stack not found, skipped" | ||
| fi | ||
|
|
||
| echo "info: deleting cluster/cluster-role/mng-role/vpc stacks" | ||
| local stacks=( | ||
| $CLUSTER-cluster | ||
| $CLUSTER-role-cluster | ||
| $CLUSTER-role-mng | ||
| $CLUSTER-vpc | ||
| ) | ||
| for stack in ${stacks[@]}; do | ||
| echo "info: deleting stack $stack" | ||
| aws cloudformation delete-stack --stack-name $stack | ||
| aws cloudformation wait stack-delete-complete --stack-name $stack | ||
| done | ||
|
|
||
| echo "info: clean missing resources" | ||
|
|
||
| # local vpcId=$(aws ec2 describe-vpcs --filter "Name=tag:Name,Values=$CLUSTER-vpc-VPC" --query 'Vpcs[0].VpcId' --output text 2>/dev/null) | ||
| # if [[ -n "$vpcId" && "$vpcId" != "None" ]]; then | ||
| # echo "info: vpc $vpcId found for cluster $CLUSTER, deleting it" | ||
| # delete_vpc $vpcId | ||
| # fi | ||
|
|
||
| # delete_role $CLUSTER-cluster-role | ||
| # delete_role $CLUSTER-mng-role | ||
|
|
||
| # echo "info: deleting keypair for node group: $CLUSTER-ssh" | ||
| # delete_ec2_key_pair $CLUSTER-ssh | ||
| } | ||
|
|
||
| # https://github.com/aws/aws-cli#other-configurable-variables | ||
| if [ -n "${AWS_REGION}" ]; then | ||
| export AWS_DEFAULT_REGION=${AWS_REGION:-} | ||
| fi | ||
|
|
||
| aws sts get-caller-identity | ||
| if [ $? -ne 0 ]; then | ||
| echo "error: failed to get caller identity" | ||
| exit 1 | ||
| fi | ||
|
|
||
| for CLUSTER in $@; do | ||
| echo "info: start to clean eks test cluster '$CLUSTER'" | ||
| clean_eks "$CLUSTER" | ||
| if [ $? -eq 0 ]; then | ||
| echo "info: succesfully cleaned the eks test cluster '$CLUSTER'" | ||
| else | ||
| echo "fatal: failed to clean the eks test cluster '$CLUSTER'" | ||
| exit 1 | ||
| fi | ||
| done |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,133 @@ | ||
| // | ||
| // Jenkins pipeline for EKS e2e job. | ||
| // | ||
| // This script is written in declarative syntax. Refer to | ||
| // https://jenkins.io/doc/book/pipeline/syntax/ for more details. | ||
| // | ||
| // Note that parameters of the job is configured in this script. | ||
| // | ||
|
|
||
| import groovy.transform.Field | ||
|
|
||
| @Field | ||
| def podYAML = ''' | ||
| apiVersion: v1 | ||
| kind: Pod | ||
| spec: | ||
| containers: | ||
| - name: main | ||
| image: gcr.io/k8s-testimages/kubekins-e2e:v20200311-1e25827-master | ||
| command: | ||
| - runner.sh | ||
| - sleep | ||
| - 1d | ||
| # we need privileged mode in order to do docker in docker | ||
| securityContext: | ||
| privileged: true | ||
| env: | ||
| - name: DOCKER_IN_DOCKER_ENABLED | ||
| value: "true" | ||
| resources: | ||
| requests: | ||
| memory: "4000Mi" | ||
| cpu: 2000m | ||
| volumeMounts: | ||
| # dind expects /var/lib/docker to be volume | ||
| - name: docker-root | ||
| mountPath: /var/lib/docker | ||
| volumes: | ||
| - name: docker-root | ||
| emptyDir: {} | ||
| ''' | ||
|
|
||
| pipeline { | ||
| agent { | ||
| kubernetes { | ||
| yaml podYAML | ||
| defaultContainer "main" | ||
| customWorkspace "/home/jenkins/agent/workspace/go/src/github.com/pingcap/tidb-operator" | ||
| } | ||
| } | ||
|
|
||
| parameters { | ||
| string(name: 'GIT_URL', defaultValue: 'git@github.com:pingcap/tidb-operator.git', description: 'git repo url') | ||
| string(name: 'GIT_REF', defaultValue: 'master', description: 'git ref spec to checkout, e.g. master, release-1.1') | ||
| string(name: 'PR_ID', defaultValue: '', description: 'pull request ID, this will override GIT_REF if set, e.g. 1889') | ||
| string(name: 'CLUSTER', defaultValue: 'jenkins-tidb-operator-e2e', description: 'the name of the cluster') | ||
| string(name: 'AWS_REGION', defaultValue: 'us-west-2', description: 'the AWS region') | ||
| string(name: 'GINKGO_NODES', defaultValue: '8', description: 'the number of ginkgo nodes') | ||
| } | ||
|
|
||
| environment { | ||
| GIT_REF = '' | ||
| ARTIFACTS = "${env.WORKSPACE}/artifacts" | ||
| } | ||
|
|
||
| stages { | ||
| stage("Prepare") { | ||
| steps { | ||
| // The declarative model for Jenkins Pipelines has a restricted | ||
| // subset of syntax that it allows in the stage blocks. We use | ||
| // script step to bypass the restriction. | ||
| // https://jenkins.io/doc/book/pipeline/syntax/#script | ||
| script { | ||
| GIT_REF = params.GIT_REF | ||
| if (params.PR_ID != "") { | ||
| GIT_REF = "refs/remotes/origin/pr/${params.PR_ID}/head" | ||
| } | ||
| } | ||
| echo "env.NODE_NAME: ${env.NODE_NAME}" | ||
| echo "env.WORKSPACE: ${env.WORKSPACE}" | ||
| echo "GIT_REF: ${GIT_REF}" | ||
| echo "ARTIFACTS: ${ARTIFACTS}" | ||
| } | ||
| } | ||
|
|
||
| stage("Checkout") { | ||
| steps { | ||
| checkout scm: [ | ||
| $class: 'GitSCM', | ||
| branches: [[name: GIT_REF]], | ||
| userRemoteConfigs: [[ | ||
| credentialsId: 'github-sre-bot-ssh', | ||
| refspec: '+refs/heads/*:refs/remotes/origin/* +refs/pull/*:refs/remotes/origin/pr/*', | ||
| url: "${params.GIT_URL}", | ||
| ]] | ||
| ] | ||
| } | ||
| } | ||
|
|
||
| stage("Run") { | ||
| steps { | ||
| withCredentials([ | ||
| string(credentialsId: 'TIDB_OPERATOR_AWS_ACCESS_KEY_ID', variable: 'AWS_ACCESS_KEY_ID'), | ||
| string(credentialsId: 'TIDB_OPERATOR_AWS_SECRET_ACCESS_KEY', variable: 'AWS_SECRET_ACCESS_KEY'), | ||
| ]) { | ||
| sh """ | ||
| #!/bin/bash | ||
| export PROVIDER=eks | ||
| export CLUSTER=${params.CLUSTER} | ||
| export AWS_REGION=${params.AWS_REGION} | ||
| export GINKGO_NODES=${params.GINKGO_NODES} | ||
| export REPORT_DIR=${ARTIFACTS} | ||
| echo "info: try to clean the cluster created previously" | ||
| ./ci/aws-clean-eks.sh \$CLUSTER | ||
| echo "info: begin to run e2e" | ||
| ./hack/e2e.sh -- --ginkgo.skip='\\[Serial\\]' --ginkgo.focus='\\[tidb-operator\\]' | ||
| """ | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| post { | ||
| always { | ||
| dir(ARTIFACTS) { | ||
| archiveArtifacts artifacts: "**", allowEmptyArchive: true | ||
| junit testResults: "*.xml", allowEmptyResults: true | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // vim: et sw=4 ts=4 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.