From ab093405c2e7ea201728902c8c26cb2e85b5ac20 Mon Sep 17 00:00:00 2001 From: Hemant Date: Fri, 27 Sep 2024 03:36:33 +0530 Subject: [PATCH] initial commit for e2e test for #6229 Signed-off-by: Hemant --- test/e2e/fqdn_dns_cache_test.go | 459 ++++++++++++++++++++++++++++++++ test/e2e/framework.go | 15 +- test/e2e/k8s_util.go | 233 ++++++++++++++++ 3 files changed, 706 insertions(+), 1 deletion(-) create mode 100644 test/e2e/fqdn_dns_cache_test.go diff --git a/test/e2e/fqdn_dns_cache_test.go b/test/e2e/fqdn_dns_cache_test.go new file mode 100644 index 00000000000..af58c7e84aa --- /dev/null +++ b/test/e2e/fqdn_dns_cache_test.go @@ -0,0 +1,459 @@ +package e2e + +import ( + crdv1beta1 "antrea.io/antrea/pkg/apis/crd/v1beta1" + agentconfig "antrea.io/antrea/pkg/config/agent" + "antrea.io/antrea/test/e2e/utils" + "context" + "encoding/json" + "fmt" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v2" + v1 "k8s.io/api/core/v1" + v12 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "strings" + "testing" + "time" +) + +/* +1) Create the KIND cluster. +2) Once cluster is up , create a service. +3) Get the IP of above service and configure the same in antrea configMap. +4) Update antrea configMap. + +5) Create NGINX deployment. +6) Get IP of one of the pods of nginx. +7) create and configure the custom CoreDNS configMap with the IP received above. +8) Create custom CoreDNS deployment. +9) Create and apply antrea FQDN policy. +10) Deploy antrea-toolbox. + + +---------- tic +11) curl the FQDN from within toolbox. +12) imitate caching the IP belonging to above FQDN resolution by keeping it in a variable. +13) edit configmap with the other IP. +14) wait for new IP to get updated in configMap and let the changes be reflected in dns pod. +15) curl the FQDN again with IP , simulating usage of cache -- and it must fail with no connectivity. +*/ + +func TestFQDNPolicyWithCachedDNS(t *testing.T) { + skipIfAntreaPolicyDisabled(t) + data, err := setupTest(t) + if err != nil { + t.Fatalf("Error when setting up test: %v", err) + } + + testFqdn := "nginx-test-pod.lfx.test" + + //TODO: Check for IPv6 ? + ipFamily := v1.IPv4Protocol + + // Create the service . + //TODO: Should the names be put up as constants instead of direct strings here? + customDnsService, err := testData.CreateUDPService("custom-dns-service", data.testNamespace, 53, 53, map[string]string{"app": "custom-dns"}, false, false, v1.ServiceTypeClusterIP, &ipFamily) + if err != nil { + t.Fatalf("Error when creating custom DNS service: %v", err) + } + require.NoError(t, err) + + // get the IP + customCoreDnsServiceObject, err := data.clientset.CoreV1().Services(data.testNamespace).Get(context.Background(), customDnsService.Name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Error when getting custom DNS service object : %v", err) + } + require.NoError(t, err) + + // Print the ClusterIP + t.Logf("ClusterIP of the service: %s\n", customCoreDnsServiceObject.Spec.ClusterIP) + + // Get Antrea ConfigMap + cm, err := data.GetAntreaConfigMap(antreaNamespace) + if err != nil { + t.Fatalf("Error when getting custom DNS configMap : %v", err) + } + require.NoError(t, err) + + // Read current value of dnsServer + var agentConf agentconfig.AgentConfig + + if err := yaml.Unmarshal([]byte(cm.Data["antrea-agent.conf"]), &agentConf); err != nil { + t.Fatalf("failed to unmarshal Agent config from ConfigMap: %v", err) + } + require.NoError(t, err) + + //Set up customDNS server IP in Antrea configmap. + agentChanges := func(config *agentconfig.AgentConfig) { + config.DNSServerOverride = customCoreDnsServiceObject.Spec.ClusterIP + } + err = data.mutateAntreaConfigMap(nil, agentChanges, false, true) + if err != nil { + t.Fatalf("Error when setting up customDNS server IP in Antrea configmap : %v", err) + } + + cm2, err := data.GetAntreaConfigMap(antreaNamespace) + if err != nil { + t.Fatalf("Error when getting custom DNS configMap : %v", err) + } + require.NoError(t, err) + + // Read current value of dnsServer + var agentConfChanged agentconfig.AgentConfig + if err := yaml.Unmarshal([]byte(cm2.Data["antrea-agent.conf"]), &agentConfChanged); err != nil { + t.Fatalf("failed to unmarshal Agent config from ConfigMap: %v", err) + } + require.NoError(t, err) + + t.Logf("dns server value set to %+v in antrea \n", agentConfChanged.DNSServerOverride) + + // Set up nginx server + nginxConfig := `events {} + +http { + server { + listen 80; + + location / { + return 200 "Pod hostname: $hostname\n"; + add_header Content-Type text/plain; + } + } +}` + + configData := map[string]string{ + "nginx.conf": nginxConfig, + } + nginxConfiMapObject, err := data.CreateConfigMap(data.testNamespace, "nginx-config", configData, nil, false) + if err != nil { + t.Fatalf("failed to create nginx ConfigMap: %v", err) + } + require.NoError(t, err) + + deploymentLabels := map[string]string{ + "app": "nginx", + } + + nginxDeployObject, err := data.CreateNginxDeploymentForTest("nginx-deployment", data.testNamespace, nginxConfiMapObject.Name, 2, deploymentLabels) + if err != nil { + t.Fatalf("failed to create nginx deployment: %v", err) + } + require.NoError(t, err) + + // Though this is used in vmagent_test.go but i think i needed it here to check for the deployment. + //TODO: Time of 15 seconds is still large for a timeout. + err = data.waitForDeploymentReady(t, data.testNamespace, nginxDeployObject.Name, 15*time.Second) + if err != nil { + t.Fatalf("error while waiting for nginx deployment to be ready : %v", err) + } + require.NoError(t, err) + + k8sUtils, err = NewKubernetesUtils(data) + if err != nil { + t.Fatalf("error getting k8s utils %+v", err) + } + require.NoError(t, err) + + nginxPods, err := k8sUtils.GetPodsByLabel(data.testNamespace, "app", "nginx") + if err != nil { + t.Fatalf("error getting Pods by label %+v", err) + } + require.NoError(t, err) + + //domainMapping holds whether the IP is mapped to Domain or not. + domainMapping := make(map[string]bool) + + // pick an IP to be added in config + var ipForConfig string + for idx, pod := range nginxPods { + //TODO: Following wait time change ? + _, err = data.podWaitForIPs(10*time.Second, pod.Name, data.testNamespace) + if err != nil { + t.Fatalf("error waiting for nginx pods to get IPs %+v", err) + } + require.NoError(t, err) + + ipStr := strings.TrimSpace(pod.Status.PodIP) + domainMapping[ipStr] = false + //pick last IP for config + if idx == len(nginxPods)-1 { + ipForConfig = ipStr + } + } + + // Create and update the custom DNS configMap + customDNSconfig := fmt.Sprintf(`lfx.test:53 { + errors + t + health + kubernetes cluster.local in-addr.arpa ip6.arpa { + pods insecure + fallthrough in-addr.arpa ip6.arpa + ttl 60 + } + hosts { + %s %s + no_reverse + pods verified + ttl 10 + } + loop + reload +}`, ipForConfig, testFqdn) + + customDNSconfigData := map[string]string{ + "Corefile": customDNSconfig, + } + + customDNSconfigMapObject, err := data.CreateConfigMap(data.testNamespace, "custom-dns-config", customDNSconfigData, nil, false) + + if err != nil { + t.Fatalf("failed to create custom dns ConfigMap: %v", err) + } + require.NoError(t, err) + domainMapping[ipForConfig] = true + + // create supporting SA, Role and Role Binding for DNS deployment. + saSpec := data.BuildServiceAccount("custom-dns-service-account", data.testNamespace, nil) + sa, err := data.CreateOrUpdateServiceAccount(saSpec) + if err != nil { + t.Fatalf("failed to create service acount for custom dns : %v", err) + } + require.NoError(t, err) + + clusterRoleSpec := &v12.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{Name: "custom-dns-role"}, + Rules: []v12.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"namespaces", "services"}, + Verbs: []string{"list", "watch"}, + }, + { + APIGroups: []string{"discovery.k8s.io"}, + Resources: []string{"endpointslices"}, + Verbs: []string{"list", "watch"}, + }, + }, + } + // TODO: Delete role on teardown. + role, err := data.CreateRole(clusterRoleSpec) + if err != nil { + t.Fatalf("failed to create cluster role for custom dns : %v", err) + } + require.NoError(t, err) + + clusterRoleBinding := &v12.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: "custom-dns-role-binding"}, + Subjects: []v12.Subject{ + { + Kind: "ServiceAccount", + Name: sa.Name, + Namespace: data.testNamespace, + }, + }, + RoleRef: v12.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: role.Name, + }, + } + + err = data.CreateRoleBinding(clusterRoleBinding) + if err != nil { + t.Fatalf("failed to create cluster role binding for custom dns : %v", err) + } + require.NoError(t, err) + + // Create custom DNS deployment + dnsDeploymentLabels := map[string]string{ + "app": "custom-dns", + } + dnsDeploymentObj, err := data.CreateCustomDnsDeployment("custom-dns-deployment", data.testNamespace, customDNSconfigMapObject.Name, sa.Name, dnsDeploymentLabels, 1) + if err != nil { + t.Fatalf("failed to create custom dns deployment : %v", err) + } + require.NoError(t, err) + + err = data.waitForDeploymentReady(t, data.testNamespace, dnsDeploymentObj.Name, 120*time.Second) + if err != nil { + t.Fatalf("error while waiting for custom dns deployment to be ready : %v", err) + } + require.NoError(t, err) + + // Create policy + npPodSelectorLabel := map[string]string{ + "app": "fqdn-cache-test", + } + port := int32(80) + builder := &utils.ClusterNetworkPolicySpecBuilder{} + builder = builder.SetName("test-acnp-fqdn"). + SetTier("application"). + SetPriority(1.0). + SetAppliedToGroup([]utils.ACNPAppliedToSpec{{PodSelector: npPodSelectorLabel}}) + builder.AddFQDNRule(testFqdn, "TCP", &port, nil, nil, "r1", nil, crdv1beta1.RuleActionAllow) + builder.AddEgress("UDP", nil, nil, nil, nil, nil, nil, nil, nil, dnsDeploymentLabels, nil, + nil, nil, nil, nil, nil, nil, crdv1beta1.RuleActionAllow, "", "", nil) + builder.AddEgress("TCP", nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, crdv1beta1.RuleActionReject, "", "", nil) + + acnp, err := k8sUtils.CreateOrUpdateACNP(builder.Get()) + failOnError(err, t) + failOnError(waitForResourceReady(t, 30*time.Second, acnp), t) + + //TODO: deletion of a namespace deletes all resources under it, do we still need to explicitly delete resources created under that namespace during a test ? + defer tearDownFQDN(t, data, builder) + + // create toolbox using their framework + toolBoxLabel := npPodSelectorLabel + pb := NewPodBuilder("toolbox", data.testNamespace, ToolboxImage) + pb.WithLabels(toolBoxLabel) + pb.WithContainerName("toolbox-container") + mutateSpecForAddingCustomDNS := func(pod *v1.Pod) { + if pod.Spec.DNSConfig == nil { + pod.Spec.DNSConfig = &v1.PodDNSConfig{} + } + pod.Spec.DNSConfig.Nameservers = []string{customCoreDnsServiceObject.Spec.ClusterIP} + + } + pb.WithMutateFunc(mutateSpecForAddingCustomDNS) + err = pb.Create(data) + if err != nil { + t.Fatalf("failed to create antrea toolbox : %v", err) + } + require.NoError(t, err) + + /* ------ Actual test ------ */ + + // idea copied from antctl_test.go:284 + // getEndpointStatus will return "Success", "Failure", or the empty string when out is not a + // marshalled metav1.Status object. + getEndpointStatus := func(out []byte) string { + var status metav1.Status + if err := json.Unmarshal(out, &status); err != nil { + // Output is not JSON or does not encode a metav1.Status object. + return "" + } + return status.Status + } + + checkFQDNaccess := func(podName, containerName, fqdn string, checkStatus bool) error { + t.Logf("trying to curl the fqdn %v", fqdn) + cmd := []string{"curl", fqdn} + stdout, stderr, err := data.RunCommandFromPod(data.testNamespace, podName, containerName, cmd) + if err != nil { + return fmt.Errorf("error when running command '%s' on Pod '%s': %v, stdout: <%v>, stderr: <%v>", strings.Join(cmd, " "), podName, err, stdout, stderr) + } + if checkStatus && getEndpointStatus([]byte(stdout)) == "Failure" { + return fmt.Errorf("failure status when accessing endpoint: <%v>", stdout) + } + fmt.Printf(" curl FQDN | running command '%s' on Pod '%s', stdout: <%v>, stderr: <%v>", strings.Join(cmd, " "), podName, stdout, stderr) + return nil + } + + err = checkFQDNaccess(pb.Name, pb.ContainerName, testFqdn, true) + if err != nil { + t.Fatalf("failed to curl FQDN from antrea toolbox on initial run : %v", err) + } + require.NoError(t, err) + + // DIG to get actual IP , to be sure. + fqdnIp, err := k8sUtils.digDnSCustom(pb.Name, pb.Namespace, testFqdn, false) + if err != nil { + t.Fatalf("failed to get IP of FQDN using DIG from toolbox pod : %v", err) + } + require.NoError(t, err) + fqdnIp = strings.TrimSpace(fqdnIp) + + t.Logf("received ip using dig for test fqdn %+v ", fqdnIp) + + var newIP string + for ip, mapped := range domainMapping { + if ip != fqdnIp && mapped == false { + newIP = ip + } + } + + t.Logf("New IP to update to DNS %v", newIP) + + // Curl the old ip and it should be a success. + err = checkFQDNaccess(pb.Name, pb.ContainerName, fqdnIp, true) + if err != nil { + t.Fatalf("failed to curl FQDN from antrea toolbox on initial run : %v", err) + } + require.NoError(t, err) + + // Create and update the custom DNS configMap + UpdatedCustomDNSconfig := fmt.Sprintf(`lfx.test:53 { + errors + t + health + kubernetes cluster.local in-addr.arpa ip6.arpa { + pods insecure + fallthrough in-addr.arpa ip6.arpa + ttl 60 + } + hosts { + %s %s + no_reverse + pods verified + ttl 10 + } + loop + reload +}`, newIP, testFqdn) + + // edit configmap with this IP + dnsConfigMap, err := data.GetConfigMap(data.testNamespace, customDNSconfigMapObject.Name) + if err != nil { + t.Fatalf("failed to get configmap to replace IP : %v", err) + } + require.NoError(t, err) + + dnsConfigMap.Data["Corefile"] = UpdatedCustomDNSconfig + err = data.UpdateConfigMap(dnsConfigMap) + if err != nil { + t.Fatalf("failed to update configmap with new IP : %v", err) + } + require.NoError(t, err) + + // Update the annotation + if dnsDeploymentObj.Annotations == nil { + dnsDeploymentObj.Annotations = make(map[string]string) + } + dnsDeploymentObj.Annotations["baar"] = "foo" + + // Get the Deployment + updatedDnsDeploymentObj, err := data.clientset.AppsV1().Deployments(data.testNamespace).Get(context.TODO(), dnsDeploymentObj.Name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Error getting Deployment: %s", err) + } + + // Update the Deployment + _, err = data.clientset.AppsV1().Deployments(data.testNamespace).Update(context.TODO(), updatedDnsDeploymentObj, metav1.UpdateOptions{}) + if err != nil { + t.Fatalf("Error updating Deployment: %s", err) + } + require.NoError(t, err) + + for { + err = checkFQDNaccess(pb.Name, pb.ContainerName, fqdnIp, true) + if err != nil { + t.Logf("curl to ip filed. Error %v", err) + break + } + // Wait for 1 second before retrying + time.Sleep(1 * time.Second) + } + + // Ensuring that the test checks that an error occurred. + require.Error(t, err) + +} + +func tearDownFQDN(t *testing.T, data *TestData, builder *utils.ClusterNetworkPolicySpecBuilder) { + // cleanup test resources + teardownTest(t, data) + failOnError(k8sUtils.DeleteACNP(builder.Name), t) +} diff --git a/test/e2e/framework.go b/test/e2e/framework.go index a497b49b53f..00927f8b2c9 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -1543,9 +1543,15 @@ func (b *PodBuilder) Create(data *TestData) error { if b.MutateFunc != nil { b.MutateFunc(pod) } - if _, err := data.clientset.CoreV1().Pods(b.Namespace).Create(context.TODO(), pod, metav1.CreateOptions{}); err != nil { + + podObj, err := data.clientset.CoreV1().Pods(b.Namespace).Create(context.TODO(), pod, metav1.CreateOptions{}) + if err != nil { return err } + _, err = data.PodWaitFor(10*time.Second, podObj.Name, podObj.Namespace, func(pod2 *corev1.Pod) (bool, error) { + return pod2.Status.Phase == corev1.PodRunning, nil + }) + return nil } @@ -1988,6 +1994,13 @@ func (data *TestData) CreateService(serviceName, namespace string, port, targetP return data.CreateServiceWithAnnotations(serviceName, namespace, port, targetPort, corev1.ProtocolTCP, selector, affinity, nodeLocalExternal, serviceType, ipFamily, annotation) } +// CreateUDPService creates a service with a UDP port and targetPort. +func (data *TestData) CreateUDPService(serviceName, namespace string, port, targetPort int32, selector map[string]string, affinity, nodeLocalExternal bool, + serviceType corev1.ServiceType, ipFamily *corev1.IPFamily) (*corev1.Service, error) { + annotation := make(map[string]string) + return data.CreateServiceWithAnnotations(serviceName, namespace, port, targetPort, corev1.ProtocolUDP, selector, affinity, nodeLocalExternal, serviceType, ipFamily, annotation) +} + // CreateServiceWithAnnotations creates a service with Annotation func (data *TestData) CreateServiceWithAnnotations(serviceName, namespace string, port, targetPort int32, protocol corev1.Protocol, selector map[string]string, affinity, nodeLocalExternal bool, serviceType corev1.ServiceType, ipFamily *corev1.IPFamily, annotations map[string]string, mutators ...func(service *corev1.Service)) (*corev1.Service, error) { diff --git a/test/e2e/k8s_util.go b/test/e2e/k8s_util.go index d665cce4c82..35df16de6cb 100644 --- a/test/e2e/k8s_util.go +++ b/test/e2e/k8s_util.go @@ -18,6 +18,7 @@ import ( "context" "errors" "fmt" + v12 "k8s.io/api/rbac/v1" "strconv" "strings" "sync" @@ -103,6 +104,7 @@ type TestNamespaceMeta struct { // GetPodByLabel returns a Pod with the matching Namespace and "pod" label if it's found. // If the pod is not found, GetPodByLabel returns "ErrPodNotFound". func (k *KubernetesUtils) GetPodByLabel(ns string, name string) (*v1.Pod, error) { + //TODO: Query why this is hard coded as "pod" pods, err := k.getPodsUncached(ns, "pod", name) if err != nil { return nil, fmt.Errorf("unable to get Pod in Namespace %s with label pod=%s: %w", ns, name, err) @@ -332,6 +334,68 @@ func decidePingProbeResult(stdout string, probeNum int) PodConnectivityMark { } return Error } +func (k *KubernetesUtils) digDnSCustom( + podName string, + podNamespace string, + dstAddr string, + useTCP bool) (string, error) { + + // Get the Pod + pod, err := k.clientset.CoreV1().Pods(podNamespace).Get(context.TODO(), podName, metav1.GetOptions{}) + if err != nil { + log.Fatalf("Error getting pod: %v", err) + } + + digCmd := fmt.Sprintf("dig %s", dstAddr) + if useTCP { + digCmd += " +tcp" + } + cmd := []string{ + "/bin/sh", + "-c", + digCmd, + } + fmt.Printf("Running: kubectl exec %s -c %s -n %s -- %s", pod.Name, pod.Spec.Containers[0].Name, pod.Namespace, strings.Join(cmd, " ")) + log.Tracef("Running: kubectl exec %s -c %s -n %s -- %s", pod.Name, pod.Spec.Containers[0].Name, pod.Namespace, strings.Join(cmd, " ")) + stdout, stderr, err := k.RunCommandFromPod(pod.Namespace, pod.Name, pod.Spec.Containers[0].Name, cmd) + fmt.Printf("%s -> %s: error when running command: err - %v /// stdout - %s /// stderr - %s", podName, dstAddr, err, stdout, stderr) + log.Tracef("%s -> %s: error when running command: err - %v /// stdout - %s /// stderr - %s", podName, dstAddr, err, stdout, stderr) + //========DiG command stdout example======== + //; <<>> DiG 9.16.6 <<>> github.com +tcp + //;; global options: +cmd + //;; Got answer: + //;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 21816 + //;; flags: qr aa rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 + // + //;; OPT PSEUDOSECTION: + //; EDNS: version: 0, flags:; udp: 4096 + //; COOKIE: 2d7fe493ea37c430 (echoed) + //;; QUESTION SECTION: + //;github.com. IN A + // + //;; ANSWER SECTION: + //github.com. 6 IN A 140.82.113.3 + // + //;; Query time: 0 msec + //;; SERVER: 10.96.0.10#53(10.96.0.10) + //;; WHEN: Tue Feb 14 22:34:23 UTC 2023 + //;; MSG SIZE rcvd: 77 + //========================================== + answerMarkIdx := strings.Index(stdout, ";; ANSWER SECTION:") + if answerMarkIdx == -1 { + return "", fmt.Errorf("failed to parse dig response") + } + splitResp := strings.Split(stdout[answerMarkIdx:], "\n") + if len(splitResp) < 2 { + return "", fmt.Errorf("failed to parse dig response") + } + ipLine := splitResp[1] + lastTab := strings.LastIndex(ipLine, "\t") + if lastTab == -1 { + return "", fmt.Errorf("failed to parse dig response") + } + return ipLine[lastTab:], nil +} func (k *KubernetesUtils) digDNS( podName string, @@ -667,6 +731,162 @@ func (data *TestData) UpdateConfigMap(configMap *v1.ConfigMap) error { return err } +func (data *TestData) CreateConfigMap(namespace, name string, configData map[string]string, binaryData map[string]byte, immutable bool) (*v1.ConfigMap, error) { + configMap := &v1.ConfigMap{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Immutable: nil, + Data: configData, + BinaryData: nil, + } + + configMapObject, err := data.clientset.CoreV1().ConfigMaps(namespace).Create(context.Background(), configMap, metav1.CreateOptions{}) + return configMapObject, err +} + +func (data *TestData) CreateNginxDeploymentForTest(name, namespace, nginxConfigMapName string, replicas int32, labels map[string]string) (*appsv1.Deployment, error) { + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "nginx", + Image: "nginx:alpine", + Ports: []v1.ContainerPort{ + { + ContainerPort: 80, + }, + }, + VolumeMounts: []v1.VolumeMount{ + { + Name: "html-volume", + MountPath: "/etc/nginx/nginx.conf", + SubPath: "nginx.conf", + }, + }, + Env: []v1.EnvVar{ + { + Name: "HOSTNAME", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, + }, + }, + }, + Volumes: []v1.Volume{ + { + Name: "html-volume", // This may be given a name here instead of passing as a parameter. + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: nginxConfigMapName, + }, + }, + }, + }, + }, + }, + }, + }, + } + + nginxDeploymentObject, err := data.clientset.AppsV1().Deployments(namespace).Create(context.Background(), deployment, metav1.CreateOptions{}) + return nginxDeploymentObject, err + +} + +func (data *TestData) CreateCustomDnsDeployment(name, namespace, configMapName, serviceAccountName string, labels map[string]string, replicas int32) (*appsv1.Deployment, error) { + + customDNSdeployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: labels, + Annotations: map[string]string{"foo": "bar"}, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + Annotations: map[string]string{"foo": "bar"}, + }, + Spec: v1.PodSpec{ + ServiceAccountName: serviceAccountName, + Containers: []v1.Container{ + { + Name: "monitor", + Image: "busybox", + Command: []string{"/bin/sh", "-c", "sleep 1d"}, + VolumeMounts: []v1.VolumeMount{ + { + Name: "config-volume", + MountPath: "/etc/coredns", + }, + }, + }, + { + Name: "coredns", + Image: "coredns/coredns:latest", + ImagePullPolicy: v1.PullIfNotPresent, + Args: []string{"-conf", "/etc/coredns/Corefile"}, + VolumeMounts: []v1.VolumeMount{ + { + Name: "config-volume", + MountPath: "/etc/coredns", + }, + }, + }, + }, + Volumes: []v1.Volume{ + { + Name: "config-volume", // This may be given a name here instead of passing as a parameter. + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: configMapName, + }, + Items: []v1.KeyToPath{ + { + Key: "Corefile", + Path: "Corefile", + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + dnsDeploymentObj, err := data.clientset.AppsV1().Deployments(namespace).Create(context.Background(), customDNSdeployment, metav1.CreateOptions{}) + return dnsDeploymentObj, err + +} + // DeleteService is a convenience function for deleting a Service by Namespace and name. func (data *TestData) DeleteService(ns, name string) error { log.Infof("Deleting Service %s in ns %s", name, ns) @@ -705,6 +925,19 @@ func (data *TestData) BuildServiceAccount(name, ns string, labels map[string]str return serviceAccount } +func (data *TestData) CreateRole(clusterRole *v12.ClusterRole) (*v12.ClusterRole, error) { + role, err := data.clientset.RbacV1().ClusterRoles().Create(context.Background(), clusterRole, metav1.CreateOptions{}) + if err != nil { + return nil, err + } + return role, nil +} + +func (data *TestData) CreateRoleBinding(roleBinding *v12.ClusterRoleBinding) error { + _, err := data.clientset.RbacV1().ClusterRoleBindings().Create(context.Background(), roleBinding, metav1.CreateOptions{}) + return err +} + // CreateOrUpdateServiceAccount is a convenience function for updating/creating ServiceAccount. func (data *TestData) CreateOrUpdateServiceAccount(sa *v1.ServiceAccount) (*v1.ServiceAccount, error) { log.Infof("Creating/updating ServiceAccount %s in ns %s", sa.Name, sa.Namespace)