From 3bcd44b6a9a23ab0ec3cdb970407729201382eaf Mon Sep 17 00:00:00 2001 From: Khare Date: Sun, 7 May 2023 22:11:44 +0530 Subject: [PATCH 01/40] indentation fixed for controller.yaml --- helm/csi-vxflexos/templates/controller.yaml | 804 ++++++++++---------- 1 file changed, 402 insertions(+), 402 deletions(-) diff --git a/helm/csi-vxflexos/templates/controller.yaml b/helm/csi-vxflexos/templates/controller.yaml index 7f3e0f39..00cca354 100644 --- a/helm/csi-vxflexos/templates/controller.yaml +++ b/helm/csi-vxflexos/templates/controller.yaml @@ -17,100 +17,100 @@ rules: verbs: ["list", "watch", "create", "update", "patch"] - apiGroups: [""] resources: ["nodes"] - {{- if hasKey .Values "podmon" }} - {{- if eq .Values.podmon.enabled true }} -verbs: ["get", "list", "watch", "patch"] - {{- else }} -verbs: ["get", "list", "watch"] - {{- end }} - {{- end }} - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - {{- if hasKey .Values "podmon" }} - {{- if eq .Values.podmon.enabled true }} -verbs: ["get", "list", "watch", "update", "patch", "delete"] - {{- else }} -verbs: ["get", "list", "watch", "update", "patch"] - {{- end }} - {{- end }} - - apiGroups: ["storage.k8s.io"] - resources: ["csinodes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments/status"] - verbs: ["patch"] - - apiGroups: ["csi.storage.k8s.io"] - resources: ["csinodeinfos"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["pods"] - {{- if hasKey .Values "podmon" }} - {{- if eq .Values.podmon.enabled true }} -verbs: ["get", "list", "watch", "update", "delete"] - {{- else }} -verbs: ["get", "list", "watch"] - {{- end }} - {{- end }} - # below for snapshotter - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - {{- if hasKey .Values "vgsnapshotter" }} - {{- if eq .Values.vgsnapshotter.enabled true }} - - apiGroups: ["volumegroup.storage.dell.com"] - resources: ["dellcsivolumegroupsnapshots","dellcsivolumegroupsnapshots/status"] - verbs: ["create", "list", "watch", "delete", "update"] - {{- end }} - {{- end }} - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - {{- if hasKey .Values "vgsnapshotter" }} - {{- if eq .Values.vgsnapshotter.enabled true }} -verbs: ["get", "list", "watch", "update", "create", "delete"] - {{- else }} -verbs: ["get", "list", "watch", "update"] - {{- end }} - {{- end }} - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots/status","volumesnapshotcontents/status"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["create", "list", "watch", "delete", "update"] - {{- if hasKey .Values.controller "replication" }} - {{- if eq .Values.controller.replication.enabled true}} - - apiGroups: ["replication.storage.dell.com"] - resources: ["dellcsireplicationgroups"] - verbs: ["create", "delete", "get", "list", "patch", "update", "watch"] - - apiGroups: ["replication.storage.dell.com"] - resources: ["dellcsireplicationgroups/status"] - verbs: ["get", "patch", "update"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["create", "delete", "get", "list", "watch", "update", "patch"] - - apiGroups: [""] - resources: ["namespaces"] - verbs: ["create", "get", "list", "watch"] - {{- end}} - {{- end}} +{{- if hasKey .Values "podmon" }} +{{- if eq .Values.podmon.enabled true }} + verbs: ["get", "list", "watch", "patch"] +{{- else }} + verbs: ["get", "list", "watch"] +{{- end }} +{{- end }} + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] +{{- if hasKey .Values "podmon" }} +{{- if eq .Values.podmon.enabled true }} + verbs: ["get", "list", "watch", "update", "patch", "delete"] +{{- else }} + verbs: ["get", "list", "watch", "update", "patch"] +{{- end }} +{{- end }} + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["pods"] +{{- if hasKey .Values "podmon" }} +{{- if eq .Values.podmon.enabled true }} + verbs: ["get", "list", "watch", "update", "delete"] +{{- else }} + verbs: ["get", "list", "watch"] +{{- end }} +{{- end }} +# below for snapshotter + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] +{{- if hasKey .Values "vgsnapshotter" }} +{{- if eq .Values.vgsnapshotter.enabled true }} + - apiGroups: ["volumegroup.storage.dell.com"] + resources: ["dellcsivolumegroupsnapshots","dellcsivolumegroupsnapshots/status"] + verbs: ["create", "list", "watch", "delete", "update"] +{{- end }} +{{- end }} + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] +{{- if hasKey .Values "vgsnapshotter" }} +{{- if eq .Values.vgsnapshotter.enabled true }} + verbs: ["get", "list", "watch", "update", "create", "delete"] +{{- else }} + verbs: ["get", "list", "watch", "update"] +{{- end }} +{{- end }} + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status","volumesnapshotcontents/status"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list", "watch", "delete", "update"] +{{- if hasKey .Values.controller "replication" }} +{{- if eq .Values.controller.replication.enabled true}} + - apiGroups: ["replication.storage.dell.com"] + resources: ["dellcsireplicationgroups"] + verbs: ["create", "delete", "get", "list", "patch", "update", "watch"] + - apiGroups: ["replication.storage.dell.com"] + resources: ["dellcsireplicationgroups/status"] + verbs: ["get", "patch", "update"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["create", "delete", "get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["create", "get", "list", "watch"] +{{- end}} +{{- end}} --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -133,318 +133,318 @@ metadata: {{- if hasKey .Values "authorization" }} {{- if eq .Values.authorization.enabled true }} annotations: - com.dell.karavi-authorization-proxy: "true" + com.dell.karavi-authorization-proxy: "true" {{ end }} {{ end }} spec: - {{- if gt (int .Values.controller.controllerCount) 1 }} -strategy: - rollingUpdate: - maxUnavailable: 1 - {{- end }} -selector: - matchLabels: - name: {{ .Release.Name }}-controller -replicas: {{ required "Must provide the number of controller instances to create." .Values.controller.controllerCount }} -template: - metadata: - labels: - {{- if eq (.Values.vgsnapshotter.enabled| toString) "true" }} - vg-snapshotter-enabled: "true" - {{- else if eq (.Values.vgsnapshotter.enabled| toString) "false" }} - vg-snapshotter-enabled: "false" - {{- end }} - name: {{ .Release.Name }}-controller - spec: - affinity: - nodeSelector: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: +{{- if gt (int .Values.controller.controllerCount) 1 }} + strategy: + rollingUpdate: + maxUnavailable: 1 +{{- end }} + selector: + matchLabels: + name: {{ .Release.Name }}-controller + replicas: {{ required "Must provide the number of controller instances to create." .Values.controller.controllerCount }} + template: + metadata: + labels: + {{- if eq (.Values.vgsnapshotter.enabled| toString) "true" }} + vg-snapshotter-enabled: "true" + {{- else if eq (.Values.vgsnapshotter.enabled| toString) "false" }} + vg-snapshotter-enabled: "false" + {{- end }} + name: {{ .Release.Name }}-controller + spec: + affinity: + nodeSelector: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchExpressions: - - key: name - operator: In - values: - - {{ .Release.Name }}-controller - topologyKey: kubernetes.io/hostname - serviceAccountName: {{ .Release.Name }}-controller - {{- if .Values.controller.nodeSelector }} - nodeSelector: - {{- toYaml .Values.controller.nodeSelector | nindent 8 }} - {{- end }} - {{- if .Values.controller.tolerations }} - tolerations: - {{- toYaml .Values.controller.tolerations | nindent 6 }} - {{- end }} - containers: - {{- if hasKey .Values "podmon" }} - {{- if eq .Values.podmon.enabled true }} - - name: podmon - image: {{ required "Must provide the podmon container image." .Values.podmon.image }} - imagePullPolicy: {{ .Values.imagePullPolicy }} - args: - {{- toYaml .Values.podmon.controller.args | nindent 12 }} - env: - - name: MY_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: MY_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: MY_POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - volumeMounts: - - name: socket-dir - mountPath: /var/run/csi - - name: vxflexos-config-params - mountPath: /vxflexos-config-params - {{- end }} - {{- end }} - - name: attacher - image: {{ required "Must provide the CSI attacher container image." ( include "csi-vxflexos.attacherImage" . ) }} - imagePullPolicy: {{ .Values.imagePullPolicy }} - args: - - "--csi-address=$(ADDRESS)" - - "--v=5" - - "--leader-election=true" - env: - - name: ADDRESS - value: /var/run/csi/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /var/run/csi - {{- if hasKey .Values.controller "replication" }} - {{- if eq .Values.controller.replication.enabled true}} - - name: dell-csi-replicator - image: {{ required "Must provide the Dell CSI Replicator image." .Values.controller.replication.image}} - imagePullPolicy: {{ .Values.imagePullPolicy }} - args: - - "--csi-address=$(ADDRESS)" - - "--leader-election=true" - - "--worker-threads=2" - - "--retry-interval-start=1s" - - "--retry-interval-max=300s" - - "--timeout=300s" - - "--context-prefix={{ .Values.controller.replication.replicationContextPrefix}}" - - "--prefix={{ .Values.controller.replication.replicationPrefix}}" - env: - - name: ADDRESS - value: /var/run/csi/csi.sock - - name: X_CSI_REPLICATION_CONFIG_DIR - value: /vxflexos-config-params - - name: X_CSI_REPLICATION_CONFIG_FILE_NAME - value: driver-config-params.yaml - volumeMounts: - - name: socket-dir - mountPath: /var/run/csi - - name: vxflexos-config-params - mountPath: /vxflexos-config-params - {{- end }} - {{- end }} - - name: provisioner - image: {{ required "Must provide the CSI provisioner container image." ( include "csi-vxflexos.provisionerImage" . ) }} - imagePullPolicy: {{ .Values.imagePullPolicy }} - args: - - "--csi-address=$(ADDRESS)" - - "--feature-gates=Topology=true" - - "--volume-name-prefix={{ required "Must provide a value to prefix to driver created volume names" .Values.controller.volumeNamePrefix }}" - - "--volume-name-uuid-length=10" - - "--leader-election=true" - - "--timeout=120s" - - "--v=5" - - "--default-fstype={{ .Values.defaultFsType | default "ext4" }}" - - "--extra-create-metadata" - env: - - name: ADDRESS - value: /var/run/csi/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /var/run/csi - {{- if hasKey .Values.controller "healthMonitor" }} - {{- if eq .Values.controller.healthMonitor.enabled true}} - - name: csi-external-health-monitor-controller - image: {{ required "Must provide the CSI external health monitor image." ( include "csi-vxflexos.healthmonitorImage" . ) }} - imagePullPolicy: {{ .Values.imagePullPolicy }} - args: - - "--csi-address=$(ADDRESS)" - - "--v=5" - - "--leader-election=true" - - "--enable-node-watcher=true" - - "--http-endpoint=:8080" - - "--monitor-interval={{ .Values.controller.healthMonitor.interval | default "60s" }}" - - "--timeout=180s" - env: - - name: ADDRESS - value: /var/run/csi/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /var/run/csi - {{- end }} - {{- end }} - {{- if hasKey .Values "vgsnapshotter" }} - {{- if eq .Values.vgsnapshotter.enabled true }} - - name: vg-snapshotter - image: {{ required "Must provide the vgsnapshotter container image." .Values.vgsnapshotter.image }} - imagePullPolicy: {{ .Values.imagePullPolicy }} - env: - - name: ADDRESS - value: /var/run/csi/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /var/run/csi - {{- end }} - {{- end }} - {{- if hasKey .Values.controller "snapshot" }} - {{- if eq .Values.controller.snapshot.enabled true }} - - name: snapshotter - image: {{ required "Must provide the CSI snapshotter container image. " ( include "csi-vxflexos.snapshotterImage" . ) }} - imagePullPolicy: {{ .Values.imagePullPolicy }} - args: - - "--csi-address=$(ADDRESS)" - - "--timeout=120s" - - "--v=5" - - "--leader-election=true" - env: - - name: ADDRESS - value: /var/run/csi/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /var/run/csi - {{- end }} - {{- end }} - {{- if hasKey .Values.controller "resizer" }} - {{- if eq .Values.controller.resizer.enabled true }} - - name: resizer - image: {{ required "Must provide the CSI resizer container image." ( include "csi-vxflexos.resizerImage" . ) }} - imagePullPolicy: {{ .Values.imagePullPolicy }} - args: - - "--csi-address=$(ADDRESS)" - - "--v=5" - - "--leader-election=true" - env: - - name: ADDRESS - value: /var/run/csi/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /var/run/csi - {{- end }} - {{- end }} - {{- if hasKey .Values "authorization" }} - {{- if eq .Values.authorization.enabled true }} - - name: karavi-authorization-proxy - image: {{ required "Must provide the authorization sidecar container image." .Values.authorization.sidecarProxyImage }} - imagePullPolicy: {{ .Values.imagePullPolicy }} - env: - - name: PROXY_HOST - value: "{{ .Values.authorization.proxyHost }}" - - name: SKIP_CERTIFICATE_VALIDATION - value: "{{ .Values.authorization.skipCertificateValidation }}" - - name: PLUGIN_IDENTIFIER - value: powerflex - - name: ACCESS_TOKEN - valueFrom: - secretKeyRef: - name: proxy-authz-tokens - key: access - - name: REFRESH_TOKEN - valueFrom: - secretKeyRef: - name: proxy-authz-tokens - key: refresh - volumeMounts: - - name: karavi-authorization-config - mountPath: /etc/karavi-authorization/config - - name: proxy-server-root-certificate - mountPath: /etc/karavi-authorization/root-certificates - - name: vxflexos-config-params - mountPath: /etc/karavi-authorization - {{- end }} - {{- end }} - - name: driver - image: "{{ required "Must provide the driver image repository." .Values.images.driverRepository }}/{{ .Chart.Name }}:{{ .Values.version }}" - imagePullPolicy: {{ .Values.imagePullPolicy }} - command: [ "/csi-vxflexos.sh" ] - args: - - "--leader-election" - - "--array-config=/vxflexos-config/config" - - "--driver-config-params=/vxflexos-config-params/driver-config-params.yaml" - env: - - name: CSI_ENDPOINT - value: /var/run/csi/csi.sock - - name: X_CSI_MODE - value: controller - - name: X_CSI_VXFLEXOS_ENABLESNAPSHOTCGDELETE - value: "{{ required "Enable this to allow deletion of all snaps in CG" .Values.enablesnapshotcgdelete }}" - - name: X_CSI_VXFLEXOS_ENABLELISTVOLUMESNAPSHOT - value: "{{ required "Enable this to have CSI ListVolumes include snapshots" .Values.enablelistvolumesnapshot }}" - - name: SSL_CERT_DIR - value: /certs - {{- if hasKey .Values.controller "replication" }} - {{- if eq .Values.controller.replication.enabled true}} - - name: X_CSI_REPLICATION_CONTEXT_PREFIX - value: {{ .Values.controller.replication.replicationContextPrefix | default "powerflex"}} - - name: X_CSI_REPLICATION_PREFIX - value: {{ .Values.controller.replication.replicationPrefix | default "replication.storage.dell.com"}} - {{- end }} - {{- end }} - {{- if hasKey .Values.controller "healthMonitor" }} - {{- if eq .Values.controller.healthMonitor.enabled true}} - - name: X_CSI_HEALTH_MONITOR_ENABLED - value: "{{ .Values.controller.healthMonitor.enabled }}" - {{- end }} - {{- end }} - {{- if hasKey .Values "nfsAcls" }} - - name: X_CSI_NFS_ACLS - value: "{{ .Values.nfsAcls }}" + - key: name + operator: In + values: + - {{ .Release.Name }}-controller + topologyKey: kubernetes.io/hostname + serviceAccountName: {{ .Release.Name }}-controller + {{- if .Values.controller.nodeSelector }} + nodeSelector: + {{- toYaml .Values.controller.nodeSelector | nindent 8 }} {{- end }} - {{- if hasKey .Values "externalAccess" }} - - name: X_CSI_POWERFLEX_EXTERNAL_ACCESS - value: "{{ .Values.externalAccess }}" + {{- if .Values.controller.tolerations }} + tolerations: + {{- toYaml .Values.controller.tolerations | nindent 6 }} {{- end }} - volumeMounts: - - name: socket-dir - mountPath: /var/run/csi - - name: vxflexos-config - mountPath: /vxflexos-config - - name: vxflexos-config-params - mountPath: /vxflexos-config-params - {{- if ge (int .Values.certSecretCount) 1 }} - - name: certs - mountPath: /certs - readOnly: true - {{- end}} -volumes: - - name: socket-dir - emptyDir: - - name: vxflexos-config - secret: - secretName: {{ .Release.Name }}-config - - name: vxflexos-config-params - configMap: - name: {{ .Release.Name }}-config-params - {{- if hasKey .Values "authorization" }} - {{- if eq .Values.authorization.enabled true }} - - name: karavi-authorization-config - secret: - secretName: karavi-authorization-config - - name: proxy-server-root-certificate - secret: - secretName: proxy-server-root-certificate - {{ end }} - {{ end }} - {{- if ge (int .Values.certSecretCount) 1 }} - - name: certs - projected: - sources: - {{- range $i, $e := until (int .Values.certSecretCount ) }} - - secret: - name: {{ print $.Release.Name "-certs-" $e }} - items: - - key: cert-{{ $e }} - path: cert-{{ $e }} - {{- end }} - {{- end }} + containers: +{{- if hasKey .Values "podmon" }} +{{- if eq .Values.podmon.enabled true }} + - name: podmon + image: {{ required "Must provide the podmon container image." .Values.podmon.image }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + args: + {{- toYaml .Values.podmon.controller.args | nindent 12 }} + env: + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: socket-dir + mountPath: /var/run/csi + - name: vxflexos-config-params + mountPath: /vxflexos-config-params +{{- end }} +{{- end }} + - name: attacher + image: {{ required "Must provide the CSI attacher container image." ( include "csi-vxflexos.attacherImage" . ) }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + args: + - "--csi-address=$(ADDRESS)" + - "--v=5" + - "--leader-election=true" + env: + - name: ADDRESS + value: /var/run/csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/run/csi + {{- if hasKey .Values.controller "replication" }} + {{- if eq .Values.controller.replication.enabled true}} + - name: dell-csi-replicator + image: {{ required "Must provide the Dell CSI Replicator image." .Values.controller.replication.image}} + imagePullPolicy: {{ .Values.imagePullPolicy }} + args: + - "--csi-address=$(ADDRESS)" + - "--leader-election=true" + - "--worker-threads=2" + - "--retry-interval-start=1s" + - "--retry-interval-max=300s" + - "--timeout=300s" + - "--context-prefix={{ .Values.controller.replication.replicationContextPrefix}}" + - "--prefix={{ .Values.controller.replication.replicationPrefix}}" + env: + - name: ADDRESS + value: /var/run/csi/csi.sock + - name: X_CSI_REPLICATION_CONFIG_DIR + value: /vxflexos-config-params + - name: X_CSI_REPLICATION_CONFIG_FILE_NAME + value: driver-config-params.yaml + volumeMounts: + - name: socket-dir + mountPath: /var/run/csi + - name: vxflexos-config-params + mountPath: /vxflexos-config-params + {{- end }} + {{- end }} + - name: provisioner + image: {{ required "Must provide the CSI provisioner container image." ( include "csi-vxflexos.provisionerImage" . ) }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + args: + - "--csi-address=$(ADDRESS)" + - "--feature-gates=Topology=true" + - "--volume-name-prefix={{ required "Must provide a value to prefix to driver created volume names" .Values.controller.volumeNamePrefix }}" + - "--volume-name-uuid-length=10" + - "--leader-election=true" + - "--timeout=120s" + - "--v=5" + - "--default-fstype={{ .Values.defaultFsType | default "ext4" }}" + - "--extra-create-metadata" + env: + - name: ADDRESS + value: /var/run/csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/run/csi + {{- if hasKey .Values.controller "healthMonitor" }} + {{- if eq .Values.controller.healthMonitor.enabled true}} + - name: csi-external-health-monitor-controller + image: {{ required "Must provide the CSI external health monitor image." ( include "csi-vxflexos.healthmonitorImage" . ) }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + args: + - "--csi-address=$(ADDRESS)" + - "--v=5" + - "--leader-election=true" + - "--enable-node-watcher=true" + - "--http-endpoint=:8080" + - "--monitor-interval={{ .Values.controller.healthMonitor.interval | default "60s" }}" + - "--timeout=180s" + env: + - name: ADDRESS + value: /var/run/csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/run/csi + {{- end }} + {{- end }} +{{- if hasKey .Values "vgsnapshotter" }} +{{- if eq .Values.vgsnapshotter.enabled true }} + - name: vg-snapshotter + image: {{ required "Must provide the vgsnapshotter container image." .Values.vgsnapshotter.image }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + env: + - name: ADDRESS + value: /var/run/csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/run/csi +{{- end }} +{{- end }} +{{- if hasKey .Values.controller "snapshot" }} +{{- if eq .Values.controller.snapshot.enabled true }} + - name: snapshotter + image: {{ required "Must provide the CSI snapshotter container image. " ( include "csi-vxflexos.snapshotterImage" . ) }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + args: + - "--csi-address=$(ADDRESS)" + - "--timeout=120s" + - "--v=5" + - "--leader-election=true" + env: + - name: ADDRESS + value: /var/run/csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/run/csi +{{- end }} +{{- end }} +{{- if hasKey .Values.controller "resizer" }} +{{- if eq .Values.controller.resizer.enabled true }} + - name: resizer + image: {{ required "Must provide the CSI resizer container image." ( include "csi-vxflexos.resizerImage" . ) }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + args: + - "--csi-address=$(ADDRESS)" + - "--v=5" + - "--leader-election=true" + env: + - name: ADDRESS + value: /var/run/csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/run/csi +{{- end }} +{{- end }} +{{- if hasKey .Values "authorization" }} +{{- if eq .Values.authorization.enabled true }} + - name: karavi-authorization-proxy + image: {{ required "Must provide the authorization sidecar container image." .Values.authorization.sidecarProxyImage }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + env: + - name: PROXY_HOST + value: "{{ .Values.authorization.proxyHost }}" + - name: SKIP_CERTIFICATE_VALIDATION + value: "{{ .Values.authorization.skipCertificateValidation }}" + - name: PLUGIN_IDENTIFIER + value: powerflex + - name: ACCESS_TOKEN + valueFrom: + secretKeyRef: + name: proxy-authz-tokens + key: access + - name: REFRESH_TOKEN + valueFrom: + secretKeyRef: + name: proxy-authz-tokens + key: refresh + volumeMounts: + - name: karavi-authorization-config + mountPath: /etc/karavi-authorization/config + - name: proxy-server-root-certificate + mountPath: /etc/karavi-authorization/root-certificates + - name: vxflexos-config-params + mountPath: /etc/karavi-authorization +{{- end }} +{{- end }} + - name: driver + image: "{{ required "Must provide the driver image repository." .Values.images.driverRepository }}/{{ .Chart.Name }}:{{ .Values.version }}" + imagePullPolicy: {{ .Values.imagePullPolicy }} + command: [ "/csi-vxflexos.sh" ] + args: + - "--leader-election" + - "--array-config=/vxflexos-config/config" + - "--driver-config-params=/vxflexos-config-params/driver-config-params.yaml" + env: + - name: CSI_ENDPOINT + value: /var/run/csi/csi.sock + - name: X_CSI_MODE + value: controller + - name: X_CSI_VXFLEXOS_ENABLESNAPSHOTCGDELETE + value: "{{ required "Enable this to allow deletion of all snaps in CG" .Values.enablesnapshotcgdelete }}" + - name: X_CSI_VXFLEXOS_ENABLELISTVOLUMESNAPSHOT + value: "{{ required "Enable this to have CSI ListVolumes include snapshots" .Values.enablelistvolumesnapshot }}" + - name: SSL_CERT_DIR + value: /certs + {{- if hasKey .Values.controller "replication" }} + {{- if eq .Values.controller.replication.enabled true}} + - name: X_CSI_REPLICATION_CONTEXT_PREFIX + value: {{ .Values.controller.replication.replicationContextPrefix | default "powerflex"}} + - name: X_CSI_REPLICATION_PREFIX + value: {{ .Values.controller.replication.replicationPrefix | default "replication.storage.dell.com"}} + {{- end }} + {{- end }} + {{- if hasKey .Values.controller "healthMonitor" }} + {{- if eq .Values.controller.healthMonitor.enabled true}} + - name: X_CSI_HEALTH_MONITOR_ENABLED + value: "{{ .Values.controller.healthMonitor.enabled }}" + {{- end }} + {{- end }} + {{- if hasKey .Values "nfsAcls" }} + - name: X_CSI_NFS_ACLS + value: "{{ .Values.nfsAcls }}" + {{- end }} + {{- if hasKey .Values "externalAccess" }} + - name: X_CSI_POWERFLEX_EXTERNAL_ACCESS + value: "{{ .Values.externalAccess }}" + {{- end }} + volumeMounts: + - name: socket-dir + mountPath: /var/run/csi + - name: vxflexos-config + mountPath: /vxflexos-config + - name: vxflexos-config-params + mountPath: /vxflexos-config-params +{{- if ge (int .Values.certSecretCount) 1 }} + - name: certs + mountPath: /certs + readOnly: true +{{- end}} + volumes: + - name: socket-dir + emptyDir: + - name: vxflexos-config + secret: + secretName: {{ .Release.Name }}-config + - name: vxflexos-config-params + configMap: + name: {{ .Release.Name }}-config-params + {{- if hasKey .Values "authorization" }} + {{- if eq .Values.authorization.enabled true }} + - name: karavi-authorization-config + secret: + secretName: karavi-authorization-config + - name: proxy-server-root-certificate + secret: + secretName: proxy-server-root-certificate + {{ end }} + {{ end }} +{{- if ge (int .Values.certSecretCount) 1 }} + - name: certs + projected: + sources: +{{- range $i, $e := until (int .Values.certSecretCount ) }} + - secret: + name: {{ print $.Release.Name "-certs-" $e }} + items: + - key: cert-{{ $e }} + path: cert-{{ $e }} +{{- end }} +{{- end }} From 39e88c6a21fc21b90a14257ad4efc98f1c8c97ab Mon Sep 17 00:00:00 2001 From: Khare Date: Sun, 7 May 2023 22:14:58 +0530 Subject: [PATCH 02/40] NFS storage class yaml updated --- samples/storageclass/storageclass-nfs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/storageclass/storageclass-nfs.yaml b/samples/storageclass/storageclass-nfs.yaml index eedbc7b5..a89568ef 100644 --- a/samples/storageclass/storageclass-nfs.yaml +++ b/samples/storageclass/storageclass-nfs.yaml @@ -92,6 +92,6 @@ volumeBindingMode: WaitForFirstConsumer # allowedTopologies helps scheduling pods on worker nodes which match all of below expressions. allowedTopologies: - matchLabelExpressions: - - key: csi-vxflexos.dellemc.com/ # Insert System ID + - key: csi-vxflexos.dellemc.com/-nfs # Insert System ID values: - csi-vxflexos.dellemc.com From e7304c8dddf021583ed48c596a3d28c35e52dd6c Mon Sep 17 00:00:00 2001 From: Khare Date: Sun, 7 May 2023 22:45:03 +0530 Subject: [PATCH 03/40] create NFS volume implemented --- service/controller.go | 657 +++++++++++++++++++++++++++--------------- service/service.go | 168 +++++++++-- 2 files changed, 575 insertions(+), 250 deletions(-) diff --git a/service/controller.go b/service/controller.go index 9c96f35b..de644939 100644 --- a/service/controller.go +++ b/service/controller.go @@ -59,6 +59,18 @@ const ( // volume create parameters map KeyMkfsFormatOption = "mkfsFormatOption" + // KeyNasName is the key used to get the NAS name from the + // volume create parameters map + KeyNasName = "nasName" + + // KeyNfsACL is the key used to get the NFS ACL from the + // volume create parameters map + KeyNfsACL = "nfsAcls" + + // KeyFsType is the key used to get the filesystem type from the + // volume create parameters map + KeyFsType = "fsType" + // DefaultVolumeSizeKiB is default volume sgolang/protobuf/blob/master/ptypesize // to create on a scaleIO cluster when no size is given, expressed in KiB DefaultVolumeSizeKiB = 16 * kiBytesInGiB @@ -83,6 +95,7 @@ const ( removeModeOnlyMe = "ONLY_ME" sioGatewayNotFound = "Not found" sioGatewayVolumeNotFound = "Could not find the volume" + sioGatewayFilesystemNotFound = "Could not find the filesystem" sioVolumeRemovalOperationInProgress = "A volume removal operation is currently in progress" sioGatewayVolumeNameInUse = "Volume name already in use. Please use a different name." errNoMultiMap = "volume not enabled for mapping to multiple hosts" @@ -135,8 +148,15 @@ func (s *service) CreateVolume( s.logStatistics() + // Check for filesystem type + isNFS := false + fsType := req.VolumeCapabilities[0].GetMount().GetFsType() + if fsType == "nfs" { + isNFS = true + } + cr := req.GetCapacityRange() - sizeInKiB, err := validateVolSize(cr) + size, err := validateVolSize(cr, isNFS) if err != nil { return nil, err } @@ -150,7 +170,6 @@ func (s *service) CreateVolume( var volumeTopology []*csi.Topology systemSegments := map[string]string{} // topology segments matching requested system for a volume if accessibility != nil && len(accessibility.GetPreferred()) > 0 { - requestedSystem := "" sID := "" system := s.systems[systemID] @@ -170,6 +189,20 @@ func (s *service) CreateVolume( constraint = tokens[1] } Log.Printf("Found topology constraint: VxFlex OS system: %s", constraint) + + // Update constraint wrt to topology specified for NFS volume + if isNFS { + nfsTokens := strings.Split(constraint, "-") + nfsLabel := "" + if len(nfsTokens) > 1 { + constraint = nfsTokens[0] + nfsLabel = nfsTokens[1] + if nfsLabel != "nfs" { + return nil, status.Errorf(codes.InvalidArgument, + "Invalid topology requested for NFS Volume. Please validate your storage class has nfs topology.") + } + } + } if constraint == sID || constraint == sName { if constraint == sID { requestedSystem = sID @@ -199,155 +232,268 @@ func (s *service) CreateVolume( } } - params = mergeStringMaps(params, req.GetSecrets()) + nfsAcls := s.opts.NfsAcls + var arr *ArrayConnectionData + sysID := s.opts.defaultSystemID + arr = s.opts.arrays[sysID] - // We require the storagePool name for creation - sp, ok := params[KeyStoragePool] - if !ok { - return nil, status.Errorf(codes.InvalidArgument, - "%s is a required parameter", KeyStoragePool) - } + if isNFS { + // fetch NAS server ID + nasName, ok := params[KeyNasName] + if !ok { + return nil, status.Errorf(codes.InvalidArgument, "`%s` is a required parameter", KeyNasName) + } + nasServerID, err := s.getNASServerIDFromName(systemID, nasName) + if err != nil { + return nil, err + } - pdID := "" - pd, ok := params[KeyProtectionDomain] - if !ok { - Log.Printf("Protection Domain name not provided; there could be conflicts if two storage pools share a name") - } else { - pdID, err = s.getProtectionDomainIDFromName(systemID, pd) + // fetch storage pool ID + pdID := "" + pd, ok := params[KeyProtectionDomain] + if !ok { + Log.Printf("Protection Domain name not provided; there could be conflicts if two storage pools share a name") + } else { + pdID, err = s.getProtectionDomainIDFromName(systemID, pd) + if err != nil { + return nil, err + } + } + + storagePoolName, ok := params[KeyStoragePool] + if !ok { + return nil, status.Errorf(codes.InvalidArgument, + "%s is a required parameter", KeyStoragePool) + } + storagePoolID, err := s.getStoragePoolID(storagePoolName, systemID, pdID) if err != nil { return nil, err } - } - volType := s.getVolProvisionType(params) // Thick or Thin + // fetch NFS ACL + if params[KeyNfsACL] != "" { + nfsAcls = params[KeyNfsACL] // Storage class takes precedence + } else if arr.NfsAcls != "" { + nfsAcls = arr.NfsAcls // Secrets next + } - name := req.GetName() - if name == "" { - return nil, status.Error(codes.InvalidArgument, - "Name cannot be empty") - } + // fetch volume name + volName := req.GetName() - if len(name) > 31 { - name = name[0:31] - Log.Printf("Requested name %s longer than 31 character max, truncated to %s\n", req.Name, name) - req.Name = name - } + // log all parameters used in CreateFilesystem call + fields := map[string]interface{}{ + "Name": volName, + "SizeInB": size, + "StoragePoolID": storagePoolID, + "NasServerID": nasServerID, + HeaderPersistentVolumeName: params[CSIPersistentVolumeName], + HeaderPersistentVolumeClaimName: params[CSIPersistentVolumeClaimName], + HeaderPersistentVolumeClaimNamespace: params[CSIPersistentVolumeClaimNamespace], + } + Log.WithFields(fields).Info("Executing CreateFilesystem with following fields") - contentSource := req.GetVolumeContentSource() - if contentSource != nil { - volumeSource := contentSource.GetVolume() - if volumeSource != nil { - Log.Printf("volume %s specified as volume content source", volumeSource.VolumeId) - return s.Clone(req, volumeSource, name, sizeInKiB, sp) + volumeParam := &siotypes.FsCreate{ + Name: volName, + SizeTotal: int(size), + StoragePoolID: storagePoolID, + NasServerID: nasServerID, } - snapshotSource := contentSource.GetSnapshot() - if snapshotSource != nil { - Log.Printf("snapshot %s specified as volume content source", snapshotSource.SnapshotId) - return s.createVolumeFromSnapshot(req, snapshotSource, name, sizeInKiB, sp) + + //Idempotency check + system, err := s.adminClients[systemID].FindSystem(systemID, "", "") + if err != nil { + return nil, err } - } + existingFS, err := system.GetFileSystemByIDName("", volName) - // TODO handle Access mode in volume capability + if existingFS != nil { + if existingFS.SizeTotal == int(size) { + vi := s.getCSIVolumeFromFilesystem(existingFS, systemID) + vi.AccessibleTopology = volumeTopology + csiResp := &csi.CreateVolumeResponse{ + Volume: vi, + } + Log.Info("Volume exists in the requested state with same size") + return csiResp, nil + } + Log.Info("'Volume name' already exists and size is different") + return nil, status.Error(codes.AlreadyExists, "'Volume name' already exists and size is different.") + } + Log.Debug("Volume does not exist, proceeding to create new volume") + fsResp, err := system.CreateFileSystem(volumeParam) + if err != nil { + Log.Debugf("Filesystem create response Error:%v", err) + return nil, status.Errorf(codes.Unknown, "Create Filesystem %s failed with error: %v", volName, err) + } - fields := map[string]interface{}{ - "name": name, - "sizeInKiB": sizeInKiB, - "storagePool": sp, - "volType": volType, - HeaderPersistentVolumeName: params[CSIPersistentVolumeName], - HeaderPersistentVolumeClaimName: params[CSIPersistentVolumeClaimName], - HeaderPersistentVolumeClaimNamespace: params[CSIPersistentVolumeClaimNamespace], - } - - Log.WithFields(fields).Info("creating volume") - - volumeParam := &siotypes.VolumeParam{ - Name: name, - VolumeSizeInKb: fmt.Sprintf("%d", sizeInKiB), - VolumeType: volType, - } - - // If the VolumeParam has a MetaData method, set the values accordingly. - if t, ok := interface{}(volumeParam).(interface { - MetaData() http.Header - }); ok { - t.MetaData().Set(HeaderPersistentVolumeName, params[CSIPersistentVolumeName]) - t.MetaData().Set(HeaderPersistentVolumeClaimName, params[CSIPersistentVolumeClaimName]) - t.MetaData().Set(HeaderPersistentVolumeClaimNamespace, params[CSIPersistentVolumeClaimNamespace]) - t.MetaData().Set(HeaderCSIPluginIdentifier, Name) - t.MetaData().Set(HeaderSystemIdentifier, systemID) + newFs, err := system.GetFileSystemByIDName(fsResp.ID, "") + if err != nil { + Log.Debugf("Find Filesystem response: %v Error: %v", newFs, err) + } + if newFs != nil { + vi := s.getCSIVolumeFromFilesystem(newFs, systemID) + vi.VolumeContext[KeyNasName] = nasName + vi.VolumeContext[KeyNfsACL] = nfsAcls + vi.VolumeContext[KeyFsType] = fsType + nfsTopology := s.GetNfsTopology(systemID) + vi.AccessibleTopology = nfsTopology + csiResp := &csi.CreateVolumeResponse{ + Volume: vi, + } + return csiResp, nil + } } else { - Log.Println("warning: goscaleio.VolumeParam: no MetaData method exists, consider updating goscaleio library.") - } + params = mergeStringMaps(params, req.GetSecrets()) - createResp, err := s.adminClients[systemID].CreateVolume(volumeParam, sp, pdID) - if err != nil { - // handle case where volume already exists - if !strings.EqualFold(err.Error(), sioGatewayVolumeNameInUse) { - Log.Printf("error creating volume: %s pool %s error: %s", name, sp, err.Error()) - return nil, status.Errorf(codes.Internal, - "error when creating volume %s storagepool %s: %s", name, sp, err.Error()) + // We require the storagePool name for creation + sp, ok := params[KeyStoragePool] + if !ok { + return nil, status.Errorf(codes.InvalidArgument, + "%s is a required parameter", KeyStoragePool) } - } - var id string - if createResp == nil { - // volume already exists, look it up by name - id, err = s.adminClients[systemID].FindVolumeID(name) - if err != nil { - return nil, status.Errorf(codes.Internal, err.Error()) + pdID := "" + pd, ok := params[KeyProtectionDomain] + if !ok { + Log.Printf("Protection Domain name not provided; there could be conflicts if two storage pools share a name") + } else { + pdID, err = s.getProtectionDomainIDFromName(systemID, pd) + if err != nil { + return nil, err + } } - } else { - id = createResp.ID - } - vol, err := s.getVolByID(id, systemID) - if err != nil { - return nil, status.Errorf(codes.Unavailable, - "error retrieving volume details: %s", err.Error()) - } - vi := s.getCSIVolume(vol, systemID) - vi.AccessibleTopology = volumeTopology + volType := s.getVolProvisionType(params) // Thick or Thin - // since the volume could have already exists, double check that the - // volume has the expected parameters - spID, err := s.getStoragePoolID(sp, systemID, pdID) - if err != nil { - return nil, status.Errorf(codes.Unavailable, - "volume exists, but could not verify parameters: %s", - err.Error()) - } - if vol.StoragePoolID != spID { - return nil, status.Errorf(codes.AlreadyExists, - "volume exists in %s, but in different storage pool than requested %s", vol.StoragePoolID, spID) - } + name := req.GetName() + if name == "" { + return nil, status.Error(codes.InvalidArgument, + "Name cannot be empty") + } - if (vi.CapacityBytes / bytesInKiB) != sizeInKiB { - return nil, status.Errorf(codes.AlreadyExists, - "volume exists, but at different size than requested") - } - copyInterestingParameters(req.GetParameters(), vi.VolumeContext) + if len(name) > 31 { + name = name[0:31] + Log.Printf("Requested name %s longer than 31 character max, truncated to %s\n", req.Name, name) + req.Name = name + } - Log.Printf("volume %s (%s) created %s\n", vi.VolumeContext["Name"], vi.VolumeId, vi.VolumeContext["CreationTime"]) + contentSource := req.GetVolumeContentSource() + if contentSource != nil { + volumeSource := contentSource.GetVolume() + if volumeSource != nil { + Log.Printf("volume %s specified as volume content source", volumeSource.VolumeId) + return s.Clone(req, volumeSource, name, size, sp) + } + snapshotSource := contentSource.GetSnapshot() + if snapshotSource != nil { + Log.Printf("snapshot %s specified as volume content source", snapshotSource.SnapshotId) + return s.createVolumeFromSnapshot(req, snapshotSource, name, size, sp) + } + } - csiResp := &csi.CreateVolumeResponse{ - Volume: vi, - } + // TODO handle Access mode in volume capability - s.clearCache() + fields := map[string]interface{}{ + "name": name, + "sizeInKiB": size, + "storagePool": sp, + "volType": volType, + HeaderPersistentVolumeName: params[CSIPersistentVolumeName], + HeaderPersistentVolumeClaimName: params[CSIPersistentVolumeClaimName], + HeaderPersistentVolumeClaimNamespace: params[CSIPersistentVolumeClaimNamespace], + } - volumeID := getVolumeIDFromCsiVolumeID(vi.VolumeId) - vol, err = s.getVolByID(volumeID, systemID) + Log.WithFields(fields).Info("creating volume") - counter := 0 + volumeParam := &siotypes.VolumeParam{ + Name: name, + VolumeSizeInKb: fmt.Sprintf("%d", size), + VolumeType: volType, + } - for err != nil && counter < 100 { - time.Sleep(3 * time.Millisecond) + // If the VolumeParam has a MetaData method, set the values accordingly. + if t, ok := interface{}(volumeParam).(interface { + MetaData() http.Header + }); ok { + t.MetaData().Set(HeaderPersistentVolumeName, params[CSIPersistentVolumeName]) + t.MetaData().Set(HeaderPersistentVolumeClaimName, params[CSIPersistentVolumeClaimName]) + t.MetaData().Set(HeaderPersistentVolumeClaimNamespace, params[CSIPersistentVolumeClaimNamespace]) + t.MetaData().Set(HeaderCSIPluginIdentifier, Name) + t.MetaData().Set(HeaderSystemIdentifier, systemID) + } else { + Log.Println("warning: goscaleio.VolumeParam: no MetaData method exists, consider updating goscaleio library.") + } + + createResp, err := s.adminClients[systemID].CreateVolume(volumeParam, sp, pdID) + if err != nil { + // handle case where volume already exists + if !strings.EqualFold(err.Error(), sioGatewayVolumeNameInUse) { + Log.Printf("error creating volume: %s pool %s error: %s", name, sp, err.Error()) + return nil, status.Errorf(codes.Internal, + "error when creating volume %s storagepool %s: %s", name, sp, err.Error()) + } + } + + var id string + if createResp == nil { + // volume already exists, look it up by name + id, err = s.adminClients[systemID].FindVolumeID(name) + if err != nil { + return nil, status.Errorf(codes.Internal, err.Error()) + } + } else { + id = createResp.ID + } + + vol, err := s.getVolByID(id, systemID) + if err != nil { + return nil, status.Errorf(codes.Unavailable, + "error retrieving volume details: %s", err.Error()) + } + vi := s.getCSIVolume(vol, systemID) + vi.AccessibleTopology = volumeTopology + + // since the volume could have already exists, double check that the + // volume has the expected parameters + spID, err := s.getStoragePoolID(sp, systemID, pdID) + if err != nil { + return nil, status.Errorf(codes.Unavailable, + "volume exists, but could not verify parameters: %s", + err.Error()) + } + if vol.StoragePoolID != spID { + return nil, status.Errorf(codes.AlreadyExists, + "volume exists in %s, but in different storage pool than requested %s", vol.StoragePoolID, spID) + } + + if (vi.CapacityBytes / bytesInKiB) != size { + return nil, status.Errorf(codes.AlreadyExists, + "volume exists, but at different size than requested") + } + copyInterestingParameters(req.GetParameters(), vi.VolumeContext) + + Log.Printf("volume %s (%s) created %s\n", vi.VolumeContext["Name"], vi.VolumeId, vi.VolumeContext["CreationTime"]) + + vi.VolumeContext[KeyFsType] = fsType + csiResp := &csi.CreateVolumeResponse{ + Volume: vi, + } + s.clearCache() + + volumeID := getVolumeIDFromCsiVolumeID(vi.VolumeId) vol, err = s.getVolByID(volumeID, systemID) - counter = counter + 1 - } - return csiResp, err + counter := 0 + + for err != nil && counter < 100 { + time.Sleep(3 * time.Millisecond) + vol, err = s.getVolByID(volumeID, systemID) + counter = counter + 1 + } + return csiResp, err + } + //return csiResp, err + return nil, status.Errorf(codes.NotFound, "Volume/Filesystem not found after create. %v", err) } // Copies the interesting parameters to the output map. @@ -503,8 +649,8 @@ func (s *service) clearCache() { // validateVolSize uses the CapacityRange range params to determine what size // volume to create, and returns an error if volume size would be greater than -// the given limit. Returned size is in KiB -func validateVolSize(cr *csi.CapacityRange) (int64, error) { +// the given limit. Returned size is in KiB for block volumes and Bytes for NFS volumes +func validateVolSize(cr *csi.CapacityRange, isNFS bool) (int64, error) { minSize := cr.GetRequiredBytes() maxSize := cr.GetLimitBytes() @@ -548,8 +694,13 @@ func validateVolSize(cr *csi.CapacityRange) (int64, error) { } } - sizeKiB = sizeGiB * kiBytesInGiB - return sizeKiB, nil + if isNFS { + sizeB = sizeGiB * kiBytesInGiB * bytesInKiB + return sizeB, nil + } else { + sizeKiB = sizeGiB * kiBytesInGiB + return sizeKiB, nil + } } func (s *service) DeleteVolume( @@ -702,139 +853,179 @@ func (s *service) ControllerPublishVolume( } - volID := getVolumeIDFromCsiVolumeID(csiVolID) - vol, err := s.getVolByID(volID, systemID) - - if err != nil { - if strings.EqualFold(err.Error(), sioGatewayVolumeNotFound) || strings.Contains(err.Error(), "must be a hexadecimal number") { - return nil, status.Error(codes.NotFound, - "volume not found") - } - return nil, status.Errorf(codes.Internal, - "failure checking volume status before controller publish: %s", - err.Error()) - } - nodeID := req.GetNodeId() if nodeID == "" { return nil, status.Error(codes.InvalidArgument, "node ID is required") } - sdcID, err := s.getSDCID(nodeID, systemID) - if err != nil { - return nil, status.Errorf(codes.NotFound, err.Error()) + // Check for NFS protocol + fsType := volumeContext[KeyFsType] + isNFS := false + if fsType == "nfs" { + isNFS = true } + if isNFS { + fsID := getFilesystemIDFromCsiVolumeID(csiVolID) + fs, err := s.getFilesystemByID(fsID, systemID) + if err != nil { + if strings.EqualFold(err.Error(), sioGatewayVolumeNotFound) || strings.Contains(err.Error(), "must be a hexadecimal number") { + return nil, status.Error(codes.NotFound, + "volume not found") + } + return nil, status.Errorf(codes.Internal, + "failure checking volume status before controller publish: %s", + err.Error()) + } - vc := req.GetVolumeCapability() - if vc == nil { - return nil, status.Error(codes.InvalidArgument, - "volume capability is required") - } + fsc := req.GetVolumeCapability() + if fsc == nil { + return nil, status.Error(codes.InvalidArgument, + "volume capability is required") + } - am := vc.GetAccessMode() - if am == nil { - return nil, status.Error(codes.InvalidArgument, - "access mode is required") - } + am := fsc.GetAccessMode() + if am == nil { + return nil, status.Error(codes.InvalidArgument, + "access mode is required") + } + if am.Mode == csi.VolumeCapability_AccessMode_UNKNOWN { + return nil, status.Error(codes.InvalidArgument, + errUnknownAccessMode) + } - if am.Mode == csi.VolumeCapability_AccessMode_UNKNOWN { - return nil, status.Error(codes.InvalidArgument, - errUnknownAccessMode) - } - // Check if volume is published to any node already - allowMultipleMappings := "FALSE" - vcs := []*csi.VolumeCapability{req.GetVolumeCapability()} - isBlock := accTypeIsBlock(vcs) + //Export for NFS + resp, err := s.exportFilesystem(ctx, req, adminClient, fs, nodeID) + return resp, err + } else { + volID := getVolumeIDFromCsiVolumeID(csiVolID) + vol, err := s.getVolByID(volID, systemID) - if len(vol.MappedSdcInfo) > 0 { - for _, sdc := range vol.MappedSdcInfo { - if sdc.SdcID == sdcID { - // TODO check if published volume is compatible with this request - // volume already mapped - Log.Debug("volume already mapped") - - // check for QoS limits of mapped volume - bandwidthLimit := volumeContext[KeyBandwidthLimitInKbps] - iopsLimit := volumeContext[KeyIopsLimit] - // validate requested QoS parameters - if err := validateQoSParameters(bandwidthLimit, iopsLimit, vol.Name); err != nil { - return nil, err - } + if err != nil { + if strings.EqualFold(err.Error(), sioGatewayVolumeNotFound) || strings.Contains(err.Error(), "must be a hexadecimal number") { + return nil, status.Error(codes.NotFound, + "volume not found") + } + return nil, status.Errorf(codes.Internal, + "failure checking volume status before controller publish: %s", + err.Error()) + } + + sdcID, err := s.getSDCID(nodeID, systemID) + if err != nil { + return nil, status.Errorf(codes.NotFound, err.Error()) + } + + vc := req.GetVolumeCapability() + if vc == nil { + return nil, status.Error(codes.InvalidArgument, + "volume capability is required") + } + + am := vc.GetAccessMode() + if am == nil { + return nil, status.Error(codes.InvalidArgument, + "access mode is required") + } + + if am.Mode == csi.VolumeCapability_AccessMode_UNKNOWN { + return nil, status.Error(codes.InvalidArgument, + errUnknownAccessMode) + } + // Check if volume is published to any node already + allowMultipleMappings := "FALSE" + vcs := []*csi.VolumeCapability{req.GetVolumeCapability()} + isBlock := accTypeIsBlock(vcs) + + if len(vol.MappedSdcInfo) > 0 { + for _, sdc := range vol.MappedSdcInfo { + if sdc.SdcID == sdcID { + // TODO check if published volume is compatible with this request + // volume already mapped + Log.Debug("volume already mapped") + + // check for QoS limits of mapped volume + bandwidthLimit := volumeContext[KeyBandwidthLimitInKbps] + iopsLimit := volumeContext[KeyIopsLimit] + // validate requested QoS parameters + if err := validateQoSParameters(bandwidthLimit, iopsLimit, vol.Name); err != nil { + return nil, err + } + + // check if volume QoS is same as requested QoS settings + if len(bandwidthLimit) > 0 && strconv.Itoa(sdc.LimitBwInMbps*1024) != bandwidthLimit { + return nil, status.Errorf(codes.InvalidArgument, + "volume %s already published with bandwidth limit: %d, but does not match the requested bandwidth limit: %s", vol.Name, sdc.LimitBwInMbps*1024, bandwidthLimit) + } else if len(iopsLimit) > 0 && strconv.Itoa(sdc.LimitIops) != iopsLimit { + return nil, status.Errorf(codes.InvalidArgument, + "volume %s already published with IOPS limit: %d, but does not match the requested IOPS limits: %s", vol.Name, sdc.LimitIops, iopsLimit) + } - // check if volume QoS is same as requested QoS settings - if len(bandwidthLimit) > 0 && strconv.Itoa(sdc.LimitBwInMbps*1024) != bandwidthLimit { - return nil, status.Errorf(codes.InvalidArgument, - "volume %s already published with bandwidth limit: %d, but does not match the requested bandwidth limit: %s", vol.Name, sdc.LimitBwInMbps*1024, bandwidthLimit) - } else if len(iopsLimit) > 0 && strconv.Itoa(sdc.LimitIops) != iopsLimit { - return nil, status.Errorf(codes.InvalidArgument, - "volume %s already published with IOPS limit: %d, but does not match the requested IOPS limits: %s", vol.Name, sdc.LimitIops, iopsLimit) + return &csi.ControllerPublishVolumeResponse{}, nil } + } - return &csi.ControllerPublishVolumeResponse{}, nil + // If volume has SINGLE_NODE cap, go no farther + switch am.Mode { + case csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, + csi.VolumeCapability_AccessMode_SINGLE_NODE_MULTI_WRITER, + csi.VolumeCapability_AccessMode_SINGLE_NODE_SINGLE_WRITER, + csi.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY: + return nil, status.Errorf(codes.FailedPrecondition, + "volume already published to SDC id: %s", vol.MappedSdcInfo[0].SdcID) } - } - // If volume has SINGLE_NODE cap, go no farther - switch am.Mode { - case csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, - csi.VolumeCapability_AccessMode_SINGLE_NODE_MULTI_WRITER, - csi.VolumeCapability_AccessMode_SINGLE_NODE_SINGLE_WRITER, - csi.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY: - return nil, status.Errorf(codes.FailedPrecondition, - "volume already published to SDC id: %s", vol.MappedSdcInfo[0].SdcID) - } + // All remaining cases are MULTI_NODE: + // This original code precludes block multi-writers, + // and is based on a faulty test that the Volume MappingToAllSdcsEnabled + // attribute must be set to allow multiple writers, which is not true. + // The proper way to control multiple mappings is with the allowMultipleMappings + // attribute passed in the MapVolumeSdcParameter. Unfortunately you cannot + // read this parameter back. - // All remaining cases are MULTI_NODE: - // This original code precludes block multi-writers, - // and is based on a faulty test that the Volume MappingToAllSdcsEnabled - // attribute must be set to allow multiple writers, which is not true. - // The proper way to control multiple mappings is with the allowMultipleMappings - // attribute passed in the MapVolumeSdcParameter. Unfortunately you cannot - // read this parameter back. + allowMultipleMappings, err = shouldAllowMultipleMappings(isBlock, am) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, err.Error()) + } - allowMultipleMappings, err = shouldAllowMultipleMappings(isBlock, am) - if err != nil { - return nil, status.Errorf(codes.InvalidArgument, err.Error()) + if err := validateAccessType(am, isBlock); err != nil { + return nil, err + } + } else { + allowMultipleMappings, err = shouldAllowMultipleMappings(isBlock, am) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, err.Error()) + } } - if err := validateAccessType(am, isBlock); err != nil { - return nil, err + mapVolumeSdcParam := &siotypes.MapVolumeSdcParam{ + SdcID: sdcID, + AllowMultipleMappings: allowMultipleMappings, + AllSdcs: "", } - } else { - allowMultipleMappings, err = shouldAllowMultipleMappings(isBlock, am) - if err != nil { - return nil, status.Errorf(codes.InvalidArgument, err.Error()) - } - } - mapVolumeSdcParam := &siotypes.MapVolumeSdcParam{ - SdcID: sdcID, - AllowMultipleMappings: allowMultipleMappings, - AllSdcs: "", - } + targetVolume := goscaleio.NewVolume(adminClient) + targetVolume.Volume = &siotypes.Volume{ID: vol.ID} - targetVolume := goscaleio.NewVolume(adminClient) - targetVolume.Volume = &siotypes.Volume{ID: vol.ID} - - err = targetVolume.MapVolumeSdc(mapVolumeSdcParam) - if err != nil { - return nil, status.Errorf(codes.Internal, - "error mapping volume to node: %s", err.Error()) - } + err = targetVolume.MapVolumeSdc(mapVolumeSdcParam) + if err != nil { + return nil, status.Errorf(codes.Internal, + "error mapping volume to node: %s", err.Error()) + } - bandwidthLimit := volumeContext[KeyBandwidthLimitInKbps] - iopsLimit := volumeContext[KeyIopsLimit] + bandwidthLimit := volumeContext[KeyBandwidthLimitInKbps] + iopsLimit := volumeContext[KeyIopsLimit] - // validate requested QoS parameters - if err := validateQoSParameters(bandwidthLimit, iopsLimit, vol.Name); err != nil { - return nil, err - } - // check for atleast one of the QoS params should exist in storage class - if len(bandwidthLimit) > 0 || len(iopsLimit) > 0 { - if err = s.setQoSParameters(ctx, systemID, sdcID, bandwidthLimit, iopsLimit, vol.Name, csiVolID, nodeID); err != nil { + // validate requested QoS parameters + if err := validateQoSParameters(bandwidthLimit, iopsLimit, vol.Name); err != nil { return nil, err } + // check for atleast one of the QoS params should exist in storage class + if len(bandwidthLimit) > 0 || len(iopsLimit) > 0 { + if err = s.setQoSParameters(ctx, systemID, sdcID, bandwidthLimit, iopsLimit, vol.Name, csiVolID, nodeID); err != nil { + return nil, err + } + } } return &csi.ControllerPublishVolumeResponse{}, nil @@ -2160,7 +2351,7 @@ func (s *service) ControllerExpandVolume(ctx context.Context, req *csi.Controlle volName := vol.Name cr := req.GetCapacityRange() Log.Printf("cr:%d", cr) - requestedSize, err := validateVolSize(cr) + requestedSize, err := validateVolSize(cr, false) if err != nil { return nil, err } diff --git a/service/service.go b/service/service.go index 5f45727c..d47ce592 100644 --- a/service/service.go +++ b/service/service.go @@ -465,7 +465,6 @@ func (s *service) checkNFS(ctx context.Context, systemID string) (bool, error) { return false, err } c := s.adminClients[systemID] - version, err := c.GetVersion() if err != nil { return false, err @@ -563,6 +562,26 @@ func (s *service) getVolByID(id string, systemID string) (*siotypes.Volume, erro return vols[0], nil } +// getFilesystemByID returns the PowerFlex filesystem from the given Powerflex filesystem ID +func (s *service) getFilesystemByID(id string, systemID string) (*siotypes.FileSystem, error) { + + adminClient := s.adminClients[systemID] + if adminClient == nil { + return nil, fmt.Errorf("can't find adminClient by id %s", systemID) + } + system, err := adminClient.FindSystem(systemID, "", "") + if err != nil { + return nil, fmt.Errorf("can't find system by id %s", systemID) + } + // The GetFileSystemByIDName API returns a filesystem, but when only passing + // in a filesystem ID or name, the response will be just the one filesystem + fs, err := system.GetFileSystemByIDName(id, "") + if err != nil { + return nil, err + } + return fs, nil +} + // getSDCID returns SDC ID from the given sdc GUID and system ID. func (s *service) getSDCID(sdcGUID string, systemID string) (string, error) { sdcGUID = strings.ToUpper(sdcGUID) @@ -621,6 +640,38 @@ func (s *service) getCSIVolume(vol *siotypes.Volume, systemID string) *csi.Volum return vi } +// getCSIVolumeFromFilesystem converts the given siotypes.FileSystem to a CSI volume +func (s *service) getCSIVolumeFromFilesystem(fs *siotypes.FileSystem, systemID string) *csi.Volume { + + // Get storage pool name; add to cache of ID to Name if not present + storagePoolName := s.getStoragePoolNameFromID(systemID, fs.StoragePoolID) + installationID, err := s.getArrayInstallationID(systemID) + if err != nil { + Log.Printf("getCSIVolumeFromFilesystem error system not found: %s with error: %v\n", systemID, err) + } + + // Make the additional volume attributes + creationTime, _ := strconv.Atoi(fs.CreationTimestamp) + attributes := map[string]string{ + "Name": fs.Name, + "StoragePoolID": fs.StoragePoolID, + "StoragePoolName": storagePoolName, + "StorageSystem": systemID, + "CreationTime": time.Unix(int64(creationTime), 0).String(), + "InstallationID": installationID, + "NasServerID": fs.NasServerID, + } + hyphen := "/" + + vi := &csi.Volume{ + VolumeId: systemID + hyphen + fs.ID, + CapacityBytes: int64(fs.SizeTotal), + VolumeContext: attributes, + } + + return vi +} + // getArryaInstallationID returns installation ID for the given system ID func (s *service) getArrayInstallationID(systemID string) (string, error) { system, err := s.adminClients[systemID].FindSystem(systemID, "", "") @@ -732,12 +783,12 @@ func getArrayConfig(ctx context.Context) (map[string]*ArrayConnectionData, error } // for PowerFlex v4.0 - var s *service - if *(c.NasName) == "" { - return nil, fmt.Errorf(fmt.Sprintf("invalid value for NasName at index %d", i)) + str := "" + if c.NasName == nil || *(c.NasName) == "" { + c.NasName = &str } if c.NfsAcls == "" { - c.NfsAcls = s.opts.NfsAcls + c.NfsAcls = str } skipCertificateValidation := c.SkipCertificateValidation || c.Insecure @@ -792,29 +843,90 @@ func getVolumeIDFromCsiVolumeID(csiVolID string) string { } err := errors.New("csiVolID unexpected string") Log.WithError(err).Errorf("%s format error", csiVolID) - return "" } -// getSystemIDFromCsiVolumeId returns PowerFlex volume ID from CSI volume ID -func (s *service) getSystemIDFromCsiVolumeID(csiVolID string) string { - i := strings.LastIndex(csiVolID, "-") - if i == -1 { +// getFilesystemIDFromCsiVolumeID returns PowerFlex filesystem ID from CSI volume ID +func getFilesystemIDFromCsiVolumeID(csiVolID string) string { + if csiVolID == "" { return "" } - tokens := strings.Split(csiVolID, "-") - if len(tokens) > 1 { - sys := csiVolID[:i] - if id, ok := s.connectedSystemNameToID[sys]; ok { - return id + containsHyphen := strings.Contains(csiVolID, "/") + if containsHyphen { + i := strings.LastIndex(csiVolID, "/") + if i == -1 { + return csiVolID + } + tokens := strings.Split(csiVolID, "/") + index := len(tokens) + if index > 0 { + return tokens[index-1] } - return sys } + err := errors.New("csiVolID unexpected string") + Log.WithError(err).Errorf("%s format error", csiVolID) + return "" +} +// getSystemIDFromCsiVolumeId returns PowerFlex volume ID from CSI volume ID +func (s *service) getSystemIDFromCsiVolumeID(csiVolID string) string { + containsHyphen := strings.Contains(csiVolID, "/") + if containsHyphen { + i := strings.LastIndex(csiVolID, "/") + if i == -1 { + return "" + } + tokens := strings.Split(csiVolID, "/") + if len(tokens) > 1 { + sys := csiVolID[:i] + if id, ok := s.connectedSystemNameToID[sys]; ok { + return id + } + return sys + } + } else { + i := strings.LastIndex(csiVolID, "-") + if i == -1 { + return "" + } + tokens := strings.Split(csiVolID, "-") + if len(tokens) > 1 { + sys := csiVolID[:i] + if id, ok := s.connectedSystemNameToID[sys]; ok { + return id + } + return sys + } + } // There is only volume ID in csi volume ID return "" } +// exportFilesystem - Method to export filesystem with idempotency +func (s *service) exportFilesystem(ctx context.Context, req *csi.ControllerPublishVolumeRequest, client *goscaleio.Client, fs *siotypes.FileSystem, nodeID string) (*csi.ControllerPublishVolumeResponse, error) { + volumeContext := req.GetVolumeContext() + if volumeContext != nil { + Log.Printf("VolumeContext:") + for key, value := range volumeContext { + Log.Printf(" [%s]=%s", key, value) + } + } + + // fetch the node IP, for exporting the FS on node. + // TODO + + // Create NFS export if it doesn't exist + // Add host IP to existing nfs export + // Add external host IP to existing nfs export + // TODO + + // create publish context + publishContext := make(map[string]string) + publishContext[KeyNasName] = volumeContext[KeyNasName] + publishContext[KeyNfsACL] = volumeContext[KeyNfsACL] + return &csi.ControllerPublishVolumeResponse{PublishContext: publishContext}, nil +} + // this function updates volumePrefixToSystems, a map of volume ID prefixes -> system IDs // this is needed for checkSystemVolumes, a function that verifies that any legacy vol ID // is found on the default system, only @@ -1075,7 +1187,7 @@ func (s *service) expandReplicationPair(ctx context.Context, req *csi.Controller Log.Printf("[expandReplicationPair] - ControllerExpandVolume expanded the remote volume first: %+v", resp) Log.Printf("[expandReplicationPair] - Ensuring remote has expanded...") - requestedSize, err := validateVolSize(req.CapacityRange) + requestedSize, err := validateVolSize(req.CapacityRange, false) if err != nil { return err } @@ -1093,3 +1205,25 @@ func (s *service) expandReplicationPair(ctx context.Context, req *csi.Controller return nil } + +func (s *service) getNASServerIDFromName(systemID, nasName string) (string, error) { + if nasName == "" { + Log.Printf("NAS server not provided.") + return "", nil + } + system, err := s.adminClients[systemID].FindSystem(systemID, "", "") + if err != nil { + return "", err + } + nas, err := system.GetNASByIDName("", nasName) + if err != nil { + return "", err + } + return nas.ID, nil +} + +func (s *service) GetNfsTopology(systemID string) []*csi.Topology { + nfsTopology := new(csi.Topology) + nfsTopology.Segments = map[string]string{Name + "/" + systemID + "-nfs": "true"} + return []*csi.Topology{nfsTopology} +} From 45e9e5cdf6218d0770f5b875180743a92b9a2fdd Mon Sep 17 00:00:00 2001 From: Khare Date: Mon, 8 May 2023 16:57:04 +0530 Subject: [PATCH 04/40] go.mod and go.sum updated --- go.mod | 2 +- go.sum | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 04aa3022..6dd31381 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/dell/dell-csi-extensions/volumeGroupSnapshot v1.2.2 github.com/dell/gocsi v1.7.0 github.com/dell/gofsutil v1.12.0 - github.com/dell/goscaleio v1.10.1-0.20230414022734-b253e33e4863 + github.com/dell/goscaleio v1.10.1-0.20230502150156-f467d3984623 github.com/fsnotify/fsnotify v1.5.1 github.com/golang/protobuf v1.5.3 github.com/google/uuid v1.3.0 diff --git a/go.sum b/go.sum index 3019cfef..d93020fe 100644 --- a/go.sum +++ b/go.sum @@ -114,12 +114,8 @@ github.com/dell/gocsi v1.7.0 h1:fMQO2zwAXCaIsUoPCcnnuPMwfQMoaI1/0aqkQVndlxU= github.com/dell/gocsi v1.7.0/go.mod h1:X/8Ll8qqKAKCenmd1gPJMUvUmgY8cK0LiS8Pck12UaU= github.com/dell/gofsutil v1.12.0 h1:oo2YHfGFKHvHS1urtqjOIKpaHwcdyqacwKHLXzUg33M= github.com/dell/gofsutil v1.12.0/go.mod h1:mGMN5grVDtHv2imNw5+gFr2RmCqeyYgBFBldUbHtV78= -github.com/dell/goscaleio v1.10.0 h1:XCEI9j+IlbqNPY7uvBjNNlT0Nt2PMXlnyxUavmennVY= -github.com/dell/goscaleio v1.10.0/go.mod h1:Zh2iQ44Jd8FMqU2h+rT5x1K6mdPMKQ15lkFxojU4z3w= -github.com/dell/goscaleio v1.10.1-0.20230413104908-fafc4b32b7ca h1:QkD1D+ROLt3hQpQ7fX8BU3adc3/CxlqjsUFmiidFv74= -github.com/dell/goscaleio v1.10.1-0.20230413104908-fafc4b32b7ca/go.mod h1:Zh2iQ44Jd8FMqU2h+rT5x1K6mdPMKQ15lkFxojU4z3w= -github.com/dell/goscaleio v1.10.1-0.20230414022734-b253e33e4863 h1:dmarfgxpB+NiuMOuOISNJGNajhFCJ6+mYUc8wrh56FQ= -github.com/dell/goscaleio v1.10.1-0.20230414022734-b253e33e4863/go.mod h1:Zh2iQ44Jd8FMqU2h+rT5x1K6mdPMKQ15lkFxojU4z3w= +github.com/dell/goscaleio v1.10.1-0.20230502150156-f467d3984623 h1:PQwO9B6aI5IF1NI9KjxRZCFm5Mk2YkB21b9SEcytmgI= +github.com/dell/goscaleio v1.10.1-0.20230502150156-f467d3984623/go.mod h1:dMTrHnXSsPus+Kd9mrs0JuyrCndoKvFP/bbEdc21Bi8= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= From d62f5a19d7f3f17befe42612328798765d9d0738 Mon Sep 17 00:00:00 2001 From: Vamsikrishna_Siddu Date: Mon, 8 May 2023 12:20:10 -0400 Subject: [PATCH 05/40] update controller publish volume for nfs. --- go.mod | 2 +- go.sum | 2 + service/controller.go | 11 ++- service/service.go | 153 ++++++++++++++++++++++++++++++++++- service/service_unit_test.go | 2 +- 5 files changed, 163 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 04aa3022..7170a340 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/dell/dell-csi-extensions/volumeGroupSnapshot v1.2.2 github.com/dell/gocsi v1.7.0 github.com/dell/gofsutil v1.12.0 - github.com/dell/goscaleio v1.10.1-0.20230414022734-b253e33e4863 + github.com/dell/goscaleio v1.10.1-0.20230503082423-9825803bafb2 github.com/fsnotify/fsnotify v1.5.1 github.com/golang/protobuf v1.5.3 github.com/google/uuid v1.3.0 diff --git a/go.sum b/go.sum index 3019cfef..c8df0b60 100644 --- a/go.sum +++ b/go.sum @@ -120,6 +120,8 @@ github.com/dell/goscaleio v1.10.1-0.20230413104908-fafc4b32b7ca h1:QkD1D+ROLt3hQ github.com/dell/goscaleio v1.10.1-0.20230413104908-fafc4b32b7ca/go.mod h1:Zh2iQ44Jd8FMqU2h+rT5x1K6mdPMKQ15lkFxojU4z3w= github.com/dell/goscaleio v1.10.1-0.20230414022734-b253e33e4863 h1:dmarfgxpB+NiuMOuOISNJGNajhFCJ6+mYUc8wrh56FQ= github.com/dell/goscaleio v1.10.1-0.20230414022734-b253e33e4863/go.mod h1:Zh2iQ44Jd8FMqU2h+rT5x1K6mdPMKQ15lkFxojU4z3w= +github.com/dell/goscaleio v1.10.1-0.20230503082423-9825803bafb2 h1:CDsgOIirCzgQY7fJ3zxHOqWepddqkqo+JDMgp5Pq/S4= +github.com/dell/goscaleio v1.10.1-0.20230503082423-9825803bafb2/go.mod h1:dMTrHnXSsPus+Kd9mrs0JuyrCndoKvFP/bbEdc21Bi8= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= diff --git a/service/controller.go b/service/controller.go index de644939..35039dc7 100644 --- a/service/controller.go +++ b/service/controller.go @@ -71,6 +71,9 @@ const ( // volume create parameters map KeyFsType = "fsType" + NFSExportLocalPath = "/" + NFSExportNamePrefix = "csishare-" + // DefaultVolumeSizeKiB is default volume sgolang/protobuf/blob/master/ptypesize // to create on a scaleIO cluster when no size is given, expressed in KiB DefaultVolumeSizeKiB = 16 * kiBytesInGiB @@ -95,6 +98,7 @@ const ( removeModeOnlyMe = "ONLY_ME" sioGatewayNotFound = "Not found" sioGatewayVolumeNotFound = "Could not find the volume" + sioGatewayNFSExportNotFound = "couldn't find NFS export" sioGatewayFilesystemNotFound = "Could not find the filesystem" sioVolumeRemovalOperationInProgress = "A volume removal operation is currently in progress" sioGatewayVolumeNameInUse = "Volume name already in use. Please use a different name." @@ -878,6 +882,11 @@ func (s *service) ControllerPublishVolume( err.Error()) } + sdcIP, err := s.getSDCIP(nodeID, systemID) + if err != nil { + return nil, status.Errorf(codes.NotFound, err.Error()) + } + fsc := req.GetVolumeCapability() if fsc == nil { return nil, status.Error(codes.InvalidArgument, @@ -895,7 +904,7 @@ func (s *service) ControllerPublishVolume( } //Export for NFS - resp, err := s.exportFilesystem(ctx, req, adminClient, fs, nodeID) + resp, err := s.exportFilesystem(ctx, req, adminClient, fs, sdcIP, nodeID, am) return resp, err } else { volID := getVolumeIDFromCsiVolumeID(csiVolID) diff --git a/service/service.go b/service/service.go index d47ce592..b45816d7 100644 --- a/service/service.go +++ b/service/service.go @@ -46,6 +46,9 @@ import ( "sigs.k8s.io/yaml" "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -599,6 +602,23 @@ func (s *service) getSDCID(sdcGUID string, systemID string) (string, error) { return id.Sdc.ID, nil } +// getSDCID returns SDC ID from the given sdc GUID and system ID. +func (s *service) getSDCIP(sdcGUID string, systemID string) (string, error) { + sdcGUID = strings.ToUpper(sdcGUID) + + // Need to translate sdcGUID to fmt.Errorf("getSDCID error systemID not found: %s", systemID) + if s.systems[systemID] == nil { + return "", fmt.Errorf("getSDCID error systemID not found: %s", systemID) + } + id, err := s.systems[systemID].FindSdc("SdcGUID", sdcGUID) + if err != nil { + return "", fmt.Errorf("error finding SDC from GUID: %s, err: %s", + sdcGUID, err.Error()) + } + + return id.Sdc.SdcIP, nil +} + // getStoragePoolID returns pool ID from the given name, system ID, and protectionDomain name func (s *service) getStoragePoolID(name, systemID, pdID string) (string, error) { @@ -903,7 +923,7 @@ func (s *service) getSystemIDFromCsiVolumeID(csiVolID string) string { } // exportFilesystem - Method to export filesystem with idempotency -func (s *service) exportFilesystem(ctx context.Context, req *csi.ControllerPublishVolumeRequest, client *goscaleio.Client, fs *siotypes.FileSystem, nodeID string) (*csi.ControllerPublishVolumeResponse, error) { +func (s *service) exportFilesystem(ctx context.Context, req *csi.ControllerPublishVolumeRequest, client *goscaleio.Client, fs *siotypes.FileSystem, nodeIP, nodeID string, am *csi.VolumeCapability_AccessMode) (*csi.ControllerPublishVolumeResponse, error) { volumeContext := req.GetVolumeContext() if volumeContext != nil { Log.Printf("VolumeContext:") @@ -916,14 +936,139 @@ func (s *service) exportFilesystem(ctx context.Context, req *csi.ControllerPubli // TODO // Create NFS export if it doesn't exist - // Add host IP to existing nfs export - // Add external host IP to existing nfs export - // TODO + + nfsExportName := NFSExportNamePrefix + fs.Name + + nfsExportExists := true + var nfsExportID string + + nfsExport, err := client.GetNFSExportByIDName(fs.ID, "") + + nfsExportID = nfsExport.ID + + if err != nil { + if strings.Contains(err.Error(), sioGatewayNFSExportNotFound) { + nfsExportExists = false + } else { + return nil, err + } + } + + if !nfsExportExists { + resp, err := client.CreateNFSExport(&siotypes.NFSExportCreate{ + Name: nfsExportName, + FileSystemID: fs.ID, + Path: NFSExportLocalPath, + }) + + if err != nil { + return nil, status.Errorf(codes.NotFound, "create NFS Export failed. Error:%v", err) + } + + nfsExportID = resp.ID + } + + //Allocate host access to NFS Share with appropriate access mode + + nfsExportResp, err := client.GetNFSExportByIDName(nfsExportID, "") + + if err != nil { + return nil, status.Errorf(codes.NotFound, "Could not find NFS Export: %s", err) + } + + readOnlyHosts := nfsExportResp.ReadOnlyHosts + readWriteHosts := nfsExportResp.ReadWriteHosts + readOnlyRootHosts := nfsExportResp.ReadOnlyRootHosts + readWriteRootHosts := nfsExportResp.ReadWriteRootHosts + + foundIncompatible := false + foundIdempotent := false + otherHostsWithAccess := len(readOnlyHosts) + + var readHostList, readWriteHostList []string + + for _, host := range readOnlyHosts { + if host == nodeIP { + foundIncompatible = true + break + } + } + + otherHostsWithAccess += len(readWriteHosts) + if !foundIncompatible { + for _, host := range readWriteHosts { + if host == nodeIP { + foundIncompatible = true + break + } + } + } + + otherHostsWithAccess += len(readOnlyRootHosts) + if !foundIncompatible { + for _, host := range readOnlyRootHosts { + readHostList = append(readHostList, host) + if host == nodeIP { + if am.Mode == csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY { + foundIdempotent = true + } else { + foundIncompatible = true + } + } + } + } + otherHostsWithAccess += len(readWriteRootHosts) + + if !foundIncompatible && !foundIdempotent { + for _, host := range readWriteRootHosts { + readWriteHostList = append(readWriteHostList, nodeIP) + if host == nodeIP { + if am.Mode == csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY { + foundIncompatible = true + } else { + foundIdempotent = true + otherHostsWithAccess-- + } + } + } + } + + if foundIncompatible { + return nil, status.Errorf(codes.NotFound, "Host: %s has access on NFS Export: %s with incompatible access mode.", nodeID, nfsExportID) + } + + if (am.Mode == csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER || am.Mode == csi.VolumeCapability_AccessMode_SINGLE_NODE_SINGLE_WRITER || am.Mode == csi.VolumeCapability_AccessMode_SINGLE_NODE_MULTI_WRITER) && otherHostsWithAccess > 0 { + return nil, status.Errorf(codes.NotFound, "Other hosts have access on NFS Share: %s", nfsExportID) + } // create publish context publishContext := make(map[string]string) publishContext[KeyNasName] = volumeContext[KeyNasName] publishContext[KeyNfsACL] = volumeContext[KeyNfsACL] + + //Idempotent case + if foundIdempotent { + Log.Info("Host has access to the given host and exists in the required state.") + return &csi.ControllerPublishVolumeResponse{PublishContext: volumeContext}, nil + } + + if am.Mode == csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY { + readHostList = append(readHostList, nodeIP) + client.ModifyNFSExport(&siotypes.NFSExportModify{AddReadOnlyRootHosts: readHostList}, nfsExportID) + } else { + readWriteHostList = append(readWriteHostList, nodeIP) + client.ModifyNFSExport(&siotypes.NFSExportModify{AddReadWriteRootHosts: readWriteHostList}, nfsExportID) + } + + if err != nil { + return nil, status.Errorf(codes.NotFound, "Allocating host %s access to NFS Export failed. Error: %v", nodeID, err) + } + Log.Debugf("NFS Export: %s is accessible to host: %s with access mode: %s", nfsExportID, nodeID, am.Mode) + Log.Debugf("ControllerPublishVolume successful for volid: [%s]", publishContext["volumeContextId"]) + // Add host IP to existing nfs export + // Add external host IP to existing nfs export + // TODO + return &csi.ControllerPublishVolumeResponse{PublishContext: publishContext}, nil } diff --git a/service/service_unit_test.go b/service/service_unit_test.go index 66eea2e9..51b85021 100644 --- a/service/service_unit_test.go +++ b/service/service_unit_test.go @@ -79,7 +79,7 @@ func TestGetVolSize(t *testing.T) { tt := tt t.Run("", func(st *testing.T) { st.Parallel() - size, err := validateVolSize(tt.cr) + size, err := validateVolSize(tt.cr, false) if tt.sizeKiB == 0 { // error is expected assert.Error(st, err) From 16731eb97e75beb863d0d8a189ef3bc5bf02a94b Mon Sep 17 00:00:00 2001 From: Vamsikrishna_Siddu Date: Mon, 8 May 2023 13:59:00 -0400 Subject: [PATCH 06/40] hardcoded volume name for testing. --- service/controller.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/service/controller.go b/service/controller.go index 35039dc7..ef54841d 100644 --- a/service/controller.go +++ b/service/controller.go @@ -161,6 +161,7 @@ func (s *service) CreateVolume( cr := req.GetCapacityRange() size, err := validateVolSize(cr, isNFS) + fmt.Println("size:", size) if err != nil { return nil, err } @@ -283,6 +284,7 @@ func (s *service) CreateVolume( // fetch volume name volName := req.GetName() + volName = "k8s-ac0ba19e9c" // log all parameters used in CreateFilesystem call fields := map[string]interface{}{ From 813497379ad73faf6cc94dfa590f8ae7d923d196 Mon Sep 17 00:00:00 2001 From: Vamsikrishna_Siddu Date: Mon, 8 May 2023 14:25:54 -0400 Subject: [PATCH 07/40] add debug. --- service/controller.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/service/controller.go b/service/controller.go index ef54841d..efb998b8 100644 --- a/service/controller.go +++ b/service/controller.go @@ -315,7 +315,11 @@ func (s *service) CreateVolume( if existingFS != nil { if existingFS.SizeTotal == int(size) { vi := s.getCSIVolumeFromFilesystem(existingFS, systemID) - vi.AccessibleTopology = volumeTopology + vi.VolumeContext[KeyNasName] = nasName + vi.VolumeContext[KeyNfsACL] = nfsAcls + vi.VolumeContext[KeyFsType] = fsType + nfsTopology := s.GetNfsTopology(systemID) + vi.AccessibleTopology = nfsTopology csiResp := &csi.CreateVolumeResponse{ Volume: vi, } From 628f423a798d17c024a0c080f962d2ddddfa691e Mon Sep 17 00:00:00 2001 From: Khare Date: Tue, 9 May 2023 23:20:58 +0530 Subject: [PATCH 08/40] files updated --- service/controller.go | 53 +++++++++++++++++++++++-------------------- service/service.go | 10 ++------ 2 files changed, 31 insertions(+), 32 deletions(-) diff --git a/service/controller.go b/service/controller.go index de644939..e02494b9 100644 --- a/service/controller.go +++ b/service/controller.go @@ -148,6 +148,8 @@ func (s *service) CreateVolume( s.logStatistics() + cr := req.GetCapacityRange() + // Check for filesystem type isNFS := false fsType := req.VolumeCapabilities[0].GetMount().GetFsType() @@ -155,12 +157,6 @@ func (s *service) CreateVolume( isNFS = true } - cr := req.GetCapacityRange() - size, err := validateVolSize(cr, isNFS) - if err != nil { - return nil, err - } - // validate AccessibleTopology accessibility := req.GetAccessibilityRequirements() if accessibility == nil { @@ -280,7 +276,10 @@ func (s *service) CreateVolume( // fetch volume name volName := req.GetName() - // log all parameters used in CreateFilesystem call + // volume size + size := cr.GetRequiredBytes() + + // log all parameters used in CreateVolume call fields := map[string]interface{}{ "Name": volName, "SizeInB": size, @@ -290,7 +289,7 @@ func (s *service) CreateVolume( HeaderPersistentVolumeClaimName: params[CSIPersistentVolumeClaimName], HeaderPersistentVolumeClaimNamespace: params[CSIPersistentVolumeClaimNamespace], } - Log.WithFields(fields).Info("Executing CreateFilesystem with following fields") + Log.WithFields(fields).Info("Executing CreateVolume with following fields") volumeParam := &siotypes.FsCreate{ Name: volName, @@ -307,6 +306,12 @@ func (s *service) CreateVolume( existingFS, err := system.GetFileSystemByIDName("", volName) if existingFS != nil { + Log.Info("Volume exists") + Log.Infof("volume size:......%v", existingFS.SizeTotal) + + Log.Infof("existingFS.SizeTotal:......%#v", existingFS.SizeTotal) + Log.Infof("given sizeInB:......%#v", size) + if existingFS.SizeTotal == int(size) { vi := s.getCSIVolumeFromFilesystem(existingFS, systemID) vi.AccessibleTopology = volumeTopology @@ -322,15 +327,17 @@ func (s *service) CreateVolume( Log.Debug("Volume does not exist, proceeding to create new volume") fsResp, err := system.CreateFileSystem(volumeParam) if err != nil { - Log.Debugf("Filesystem create response Error:%v", err) - return nil, status.Errorf(codes.Unknown, "Create Filesystem %s failed with error: %v", volName, err) + Log.Debugf("Volume create response Error:%v", err) + return nil, status.Errorf(codes.Unknown, "Create Volume %s failed with error: %v", volName, err) } + Log.Infof("FS ID.....: %v", fsResp.ID) newFs, err := system.GetFileSystemByIDName(fsResp.ID, "") if err != nil { - Log.Debugf("Find Filesystem response: %v Error: %v", newFs, err) + Log.Debugf("Find Volume response: %v Error: %v", newFs, err) } if newFs != nil { + Log.Infof("newFs.SizeTotal.... %v", newFs.SizeTotal) vi := s.getCSIVolumeFromFilesystem(newFs, systemID) vi.VolumeContext[KeyNasName] = nasName vi.VolumeContext[KeyNfsACL] = nfsAcls @@ -343,6 +350,11 @@ func (s *service) CreateVolume( return csiResp, nil } } else { + size, err := validateVolSize(cr) + if err != nil { + return nil, err + } + params = mergeStringMaps(params, req.GetSecrets()) // We require the storagePool name for creation @@ -403,7 +415,7 @@ func (s *service) CreateVolume( HeaderPersistentVolumeClaimNamespace: params[CSIPersistentVolumeClaimNamespace], } - Log.WithFields(fields).Info("creating volume") + Log.WithFields(fields).Info("Executing CreateVolume with following fields") volumeParam := &siotypes.VolumeParam{ Name: name, @@ -649,12 +661,11 @@ func (s *service) clearCache() { // validateVolSize uses the CapacityRange range params to determine what size // volume to create, and returns an error if volume size would be greater than -// the given limit. Returned size is in KiB for block volumes and Bytes for NFS volumes -func validateVolSize(cr *csi.CapacityRange, isNFS bool) (int64, error) { +// the given limit. Returned size is in KiB +func validateVolSize(cr *csi.CapacityRange) (int64, error) { minSize := cr.GetRequiredBytes() maxSize := cr.GetLimitBytes() - if minSize < 0 || maxSize < 0 { return 0, status.Errorf( codes.OutOfRange, @@ -694,13 +705,8 @@ func validateVolSize(cr *csi.CapacityRange, isNFS bool) (int64, error) { } } - if isNFS { - sizeB = sizeGiB * kiBytesInGiB * bytesInKiB - return sizeB, nil - } else { - sizeKiB = sizeGiB * kiBytesInGiB - return sizeKiB, nil - } + sizeKiB = sizeGiB * kiBytesInGiB + return sizeKiB, nil } func (s *service) DeleteVolume( @@ -893,7 +899,6 @@ func (s *service) ControllerPublishVolume( return nil, status.Error(codes.InvalidArgument, errUnknownAccessMode) } - //Export for NFS resp, err := s.exportFilesystem(ctx, req, adminClient, fs, nodeID) return resp, err @@ -2351,7 +2356,7 @@ func (s *service) ControllerExpandVolume(ctx context.Context, req *csi.Controlle volName := vol.Name cr := req.GetCapacityRange() Log.Printf("cr:%d", cr) - requestedSize, err := validateVolSize(cr, false) + requestedSize, err := validateVolSize(cr) if err != nil { return nil, err } diff --git a/service/service.go b/service/service.go index d47ce592..453a63db 100644 --- a/service/service.go +++ b/service/service.go @@ -905,14 +905,8 @@ func (s *service) getSystemIDFromCsiVolumeID(csiVolID string) string { // exportFilesystem - Method to export filesystem with idempotency func (s *service) exportFilesystem(ctx context.Context, req *csi.ControllerPublishVolumeRequest, client *goscaleio.Client, fs *siotypes.FileSystem, nodeID string) (*csi.ControllerPublishVolumeResponse, error) { volumeContext := req.GetVolumeContext() - if volumeContext != nil { - Log.Printf("VolumeContext:") - for key, value := range volumeContext { - Log.Printf(" [%s]=%s", key, value) - } - } - // fetch the node IP, for exporting the FS on node. + // fetch the node IP, for exporting the FS on host. // TODO // Create NFS export if it doesn't exist @@ -1187,7 +1181,7 @@ func (s *service) expandReplicationPair(ctx context.Context, req *csi.Controller Log.Printf("[expandReplicationPair] - ControllerExpandVolume expanded the remote volume first: %+v", resp) Log.Printf("[expandReplicationPair] - Ensuring remote has expanded...") - requestedSize, err := validateVolSize(req.CapacityRange, false) + requestedSize, err := validateVolSize(req.CapacityRange) if err != nil { return err } From c9e6068e297a9ffa977416d8b291d92db5838bac Mon Sep 17 00:00:00 2001 From: Vamsikrishna_Siddu Date: Wed, 10 May 2023 00:09:19 -0400 Subject: [PATCH 09/40] add debug. --- service/controller.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/service/controller.go b/service/controller.go index efb998b8..2a60caf4 100644 --- a/service/controller.go +++ b/service/controller.go @@ -161,6 +161,7 @@ func (s *service) CreateVolume( cr := req.GetCapacityRange() size, err := validateVolSize(cr, isNFS) + fmt.Println("isNFS:", isNFS) fmt.Println("size:", size) if err != nil { return nil, err @@ -249,6 +250,7 @@ func (s *service) CreateVolume( return nil, status.Errorf(codes.InvalidArgument, "`%s` is a required parameter", KeyNasName) } nasServerID, err := s.getNASServerIDFromName(systemID, nasName) + fmt.Println("nasSeverID:", nasServerID) if err != nil { return nil, err } @@ -263,6 +265,7 @@ func (s *service) CreateVolume( if err != nil { return nil, err } + fmt.Println("pdID:", pdID) } storagePoolName, ok := params[KeyStoragePool] @@ -275,6 +278,8 @@ func (s *service) CreateVolume( return nil, err } + fmt.Println("storagePoolID:", storagePoolID) + // fetch NFS ACL if params[KeyNfsACL] != "" { nfsAcls = params[KeyNfsACL] // Storage class takes precedence @@ -282,9 +287,12 @@ func (s *service) CreateVolume( nfsAcls = arr.NfsAcls // Secrets next } + fmt.Println("nfsScls:", nfsAcls) + // fetch volume name volName := req.GetName() - volName = "k8s-ac0ba19e9c" + + fmt.Println("volName:", volName) // log all parameters used in CreateFilesystem call fields := map[string]interface{}{ @@ -305,6 +313,8 @@ func (s *service) CreateVolume( NasServerID: nasServerID, } + fmt.Printf("volumeParam:%#v\n", volumeParam) + //Idempotency check system, err := s.adminClients[systemID].FindSystem(systemID, "", "") if err != nil { @@ -331,6 +341,7 @@ func (s *service) CreateVolume( } Log.Debug("Volume does not exist, proceeding to create new volume") fsResp, err := system.CreateFileSystem(volumeParam) + fmt.Println("fsResp:", fsResp) if err != nil { Log.Debugf("Filesystem create response Error:%v", err) return nil, status.Errorf(codes.Unknown, "Create Filesystem %s failed with error: %v", volName, err) From 2d7fd6c717223031c739aebca2310386f75ff24d Mon Sep 17 00:00:00 2001 From: Khare Date: Wed, 10 May 2023 11:45:03 +0530 Subject: [PATCH 10/40] removed debug statements --- service/controller.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/service/controller.go b/service/controller.go index e02494b9..04e7dc1f 100644 --- a/service/controller.go +++ b/service/controller.go @@ -306,12 +306,6 @@ func (s *service) CreateVolume( existingFS, err := system.GetFileSystemByIDName("", volName) if existingFS != nil { - Log.Info("Volume exists") - Log.Infof("volume size:......%v", existingFS.SizeTotal) - - Log.Infof("existingFS.SizeTotal:......%#v", existingFS.SizeTotal) - Log.Infof("given sizeInB:......%#v", size) - if existingFS.SizeTotal == int(size) { vi := s.getCSIVolumeFromFilesystem(existingFS, systemID) vi.AccessibleTopology = volumeTopology @@ -327,17 +321,15 @@ func (s *service) CreateVolume( Log.Debug("Volume does not exist, proceeding to create new volume") fsResp, err := system.CreateFileSystem(volumeParam) if err != nil { - Log.Debugf("Volume create response Error:%v", err) + Log.Debugf("Create volume response error:%v", err) return nil, status.Errorf(codes.Unknown, "Create Volume %s failed with error: %v", volName, err) } - Log.Infof("FS ID.....: %v", fsResp.ID) newFs, err := system.GetFileSystemByIDName(fsResp.ID, "") if err != nil { Log.Debugf("Find Volume response: %v Error: %v", newFs, err) } if newFs != nil { - Log.Infof("newFs.SizeTotal.... %v", newFs.SizeTotal) vi := s.getCSIVolumeFromFilesystem(newFs, systemID) vi.VolumeContext[KeyNasName] = nasName vi.VolumeContext[KeyNfsACL] = nfsAcls From cf0b1eac14deec071d2bc57296df569ee746059d Mon Sep 17 00:00:00 2001 From: Vamsikrishna_Siddu Date: Wed, 10 May 2023 02:32:58 -0400 Subject: [PATCH 11/40] add debug. --- service/controller.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/service/controller.go b/service/controller.go index 308f4aa4..c0e496d7 100644 --- a/service/controller.go +++ b/service/controller.go @@ -285,6 +285,7 @@ func (s *service) CreateVolume( // fetch volume name volName := req.GetName() + volName = "tweeFS3" fmt.Println("volName:", volName) @@ -893,6 +894,7 @@ func (s *service) ControllerPublishVolume( } if isNFS { fsID := getFilesystemIDFromCsiVolumeID(csiVolID) + fmt.Println("fsID:", fsID) fs, err := s.getFilesystemByID(fsID, systemID) if err != nil { if strings.EqualFold(err.Error(), sioGatewayVolumeNotFound) || strings.Contains(err.Error(), "must be a hexadecimal number") { From 3417c105ec0d55e450233532fab2e644564c001a Mon Sep 17 00:00:00 2001 From: Vamsikrishna_Siddu Date: Wed, 10 May 2023 02:54:06 -0400 Subject: [PATCH 12/40] fixed error. --- service/service.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/service/service.go b/service/service.go index 2496e504..ff0458bf 100644 --- a/service/service.go +++ b/service/service.go @@ -938,16 +938,17 @@ func (s *service) exportFilesystem(ctx context.Context, req *csi.ControllerPubli nfsExport, err := client.GetNFSExportByIDName(fs.ID, "") - nfsExportID = nfsExport.ID - if err != nil { if strings.Contains(err.Error(), sioGatewayNFSExportNotFound) { nfsExportExists = false + } else { return nil, err } } + nfsExportID = nfsExport.ID + if !nfsExportExists { resp, err := client.CreateNFSExport(&siotypes.NFSExportCreate{ Name: nfsExportName, From 70712933fc8d9b285c3d60eb9452248e2642923d Mon Sep 17 00:00:00 2001 From: Vamsikrishna_Siddu Date: Wed, 10 May 2023 03:07:40 -0400 Subject: [PATCH 13/40] fix the panic. --- service/service.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/service/service.go b/service/service.go index ff0458bf..4eebdf29 100644 --- a/service/service.go +++ b/service/service.go @@ -947,8 +947,6 @@ func (s *service) exportFilesystem(ctx context.Context, req *csi.ControllerPubli } } - nfsExportID = nfsExport.ID - if !nfsExportExists { resp, err := client.CreateNFSExport(&siotypes.NFSExportCreate{ Name: nfsExportName, @@ -961,6 +959,9 @@ func (s *service) exportFilesystem(ctx context.Context, req *csi.ControllerPubli } nfsExportID = resp.ID + } else { + nfsExportID = nfsExport.ID + } //Allocate host access to NFS Share with appropriate access mode From 678d3de4f2d0bbcd10387dfc8e96f8ea6f1519eb Mon Sep 17 00:00:00 2001 From: Vamsikrishna_Siddu Date: Wed, 10 May 2023 07:04:53 -0400 Subject: [PATCH 14/40] add debug. --- service/node.go | 1 + service/service.go | 3 +++ 2 files changed, 4 insertions(+) diff --git a/service/node.go b/service/node.go index 2167e823..5d4e1c7c 100644 --- a/service/node.go +++ b/service/node.go @@ -650,6 +650,7 @@ func (s *service) NodeGetInfo( // Create the topology keys // csi-vxflexos.dellemc.com/: topology := map[string]string{} + fmt.Printf("%#v", connectedSystemID) for _, sysID := range connectedSystemID { isNFS, err := s.checkNFS(ctx, sysID) if err != nil { diff --git a/service/service.go b/service/service.go index 4eebdf29..d1c36595 100644 --- a/service/service.go +++ b/service/service.go @@ -467,7 +467,10 @@ func (s *service) checkNFS(ctx context.Context, systemID string) (bool, error) { if err != nil { return false, err } + c := s.adminClients[systemID] + fmt.Printf("adminClients:%#v\n", s.adminClients) + fmt.Printf("client: %#v\n", c) version, err := c.GetVersion() if err != nil { return false, err From 46dba7140cb2a2308ff8b9e7cc190e14920cf130 Mon Sep 17 00:00:00 2001 From: Vamsikrishna_Siddu Date: Wed, 10 May 2023 07:17:09 -0400 Subject: [PATCH 15/40] fixed the nil panic. --- service/service.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/service/service.go b/service/service.go index d1c36595..9765fa7d 100644 --- a/service/service.go +++ b/service/service.go @@ -471,6 +471,9 @@ func (s *service) checkNFS(ctx context.Context, systemID string) (bool, error) { c := s.adminClients[systemID] fmt.Printf("adminClients:%#v\n", s.adminClients) fmt.Printf("client: %#v\n", c) + if c == nil { + return false, nil + } version, err := c.GetVersion() if err != nil { return false, err From df0ed7273be4e73d1c515e02804a425f8ecec87b Mon Sep 17 00:00:00 2001 From: Vamsikrishna_Siddu Date: Wed, 10 May 2023 08:09:21 -0400 Subject: [PATCH 16/40] fixed the nfs export. --- service/service.go | 1 - 1 file changed, 1 deletion(-) diff --git a/service/service.go b/service/service.go index 9765fa7d..ab7ffb3f 100644 --- a/service/service.go +++ b/service/service.go @@ -957,7 +957,6 @@ func (s *service) exportFilesystem(ctx context.Context, req *csi.ControllerPubli resp, err := client.CreateNFSExport(&siotypes.NFSExportCreate{ Name: nfsExportName, FileSystemID: fs.ID, - Path: NFSExportLocalPath, }) if err != nil { From 3f6b7793eabcf13458acf019bfe7fd70ecd301d0 Mon Sep 17 00:00:00 2001 From: Vamsikrishna_Siddu Date: Wed, 10 May 2023 08:43:37 -0400 Subject: [PATCH 17/40] fix nfs export error. --- service/service.go | 1 + 1 file changed, 1 insertion(+) diff --git a/service/service.go b/service/service.go index ab7ffb3f..87dbe4ec 100644 --- a/service/service.go +++ b/service/service.go @@ -957,6 +957,7 @@ func (s *service) exportFilesystem(ctx context.Context, req *csi.ControllerPubli resp, err := client.CreateNFSExport(&siotypes.NFSExportCreate{ Name: nfsExportName, FileSystemID: fs.ID, + Path: NFSExportLocalPath + fs.Name, }) if err != nil { From e6a708393fc4bff7480ea98638202a159cb1fcb9 Mon Sep 17 00:00:00 2001 From: Khare Date: Wed, 10 May 2023 20:22:18 +0530 Subject: [PATCH 18/40] files updated --- service/controller.go | 10 +++++++++- service/service.go | 3 +++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/service/controller.go b/service/controller.go index 04e7dc1f..9b1106a3 100644 --- a/service/controller.go +++ b/service/controller.go @@ -95,7 +95,6 @@ const ( removeModeOnlyMe = "ONLY_ME" sioGatewayNotFound = "Not found" sioGatewayVolumeNotFound = "Could not find the volume" - sioGatewayFilesystemNotFound = "Could not find the filesystem" sioVolumeRemovalOperationInProgress = "A volume removal operation is currently in progress" sioGatewayVolumeNameInUse = "Volume name already in use. Please use a different name." errNoMultiMap = "volume not enabled for mapping to multiple hosts" @@ -228,6 +227,15 @@ func (s *service) CreateVolume( } } + if req.VolumeCapabilities[0].GetBlock() != nil { + // We need to check if user requests raw block access from nfs and prevent that + fsType, ok := params[KeyFsType] + // FsType can be empty + if ok && fsType == "nfs" { + return nil, status.Errorf(codes.InvalidArgument, "raw block requested from NFS Volume") + } + } + nfsAcls := s.opts.NfsAcls var arr *ArrayConnectionData sysID := s.opts.defaultSystemID diff --git a/service/service.go b/service/service.go index 453a63db..a7cd1fd1 100644 --- a/service/service.go +++ b/service/service.go @@ -465,6 +465,9 @@ func (s *service) checkNFS(ctx context.Context, systemID string) (bool, error) { return false, err } c := s.adminClients[systemID] + if c == nil { + return false, nil + } version, err := c.GetVersion() if err != nil { return false, err From 8fbdd070d419ce315d45fcb49634506f8307df26 Mon Sep 17 00:00:00 2001 From: Vamsikrishna_Siddu Date: Thu, 11 May 2023 01:44:33 -0400 Subject: [PATCH 19/40] updated files. --- service/controller.go | 17 +++++++++++++---- service/service.go | 34 +++++++++------------------------- service/service_unit_test.go | 2 +- 3 files changed, 23 insertions(+), 30 deletions(-) diff --git a/service/controller.go b/service/controller.go index 1b2daedb..d6e57a1e 100644 --- a/service/controller.go +++ b/service/controller.go @@ -849,7 +849,14 @@ func (s *service) ControllerPublishVolume( } } + // create publish context + publishContext := make(map[string]string) + publishContext[KeyNasName] = volumeContext[KeyNasName] + publishContext[KeyNfsACL] = volumeContext[KeyNfsACL] + csiVolID := req.GetVolumeId() + publishContext["volumeContextId"] = csiVolID + if csiVolID == "" { return nil, status.Error(codes.InvalidArgument, "volume ID is required") @@ -898,12 +905,12 @@ func (s *service) ControllerPublishVolume( fmt.Println("fsID:", fsID) fs, err := s.getFilesystemByID(fsID, systemID) if err != nil { - if strings.EqualFold(err.Error(), sioGatewayVolumeNotFound) || strings.Contains(err.Error(), "must be a hexadecimal number") { + if strings.EqualFold(err.Error(), sioGatewayFilesystemNotFound) || strings.Contains(err.Error(), "must be a hexadecimal number") { return nil, status.Error(codes.NotFound, - "volume not found") + "filesystem not found") } return nil, status.Errorf(codes.Internal, - "failure checking volume status before controller publish: %s", + "failure checking filesystem status before controller publish: %s", err.Error()) } @@ -912,6 +919,8 @@ func (s *service) ControllerPublishVolume( return nil, status.Errorf(codes.NotFound, err.Error()) } + publishContext["host"] = sdcIP + fsc := req.GetVolumeCapability() if fsc == nil { return nil, status.Error(codes.InvalidArgument, @@ -928,7 +937,7 @@ func (s *service) ControllerPublishVolume( errUnknownAccessMode) } //Export for NFS - resp, err := s.exportFilesystem(ctx, req, adminClient, fs, sdcIP, nodeID, am) + resp, err := s.exportFilesystem(ctx, req, adminClient, fs, sdcIP, nodeID, publishContext, am) return resp, err } else { volID := getVolumeIDFromCsiVolumeID(csiVolID) diff --git a/service/service.go b/service/service.go index 88f95771..dcc087c3 100644 --- a/service/service.go +++ b/service/service.go @@ -613,9 +613,9 @@ func (s *service) getSDCID(sdcGUID string, systemID string) (string, error) { func (s *service) getSDCIP(sdcGUID string, systemID string) (string, error) { sdcGUID = strings.ToUpper(sdcGUID) - // Need to translate sdcGUID to fmt.Errorf("getSDCID error systemID not found: %s", systemID) + // Need to translate sdcGUID to fmt.Errorf("getSDCIP error systemID not found: %s", systemID) if s.systems[systemID] == nil { - return "", fmt.Errorf("getSDCID error systemID not found: %s", systemID) + return "", fmt.Errorf("getSDCIP error systemID not found: %s", systemID) } id, err := s.systems[systemID].FindSdc("SdcGUID", sdcGUID) if err != nil { @@ -930,19 +930,13 @@ func (s *service) getSystemIDFromCsiVolumeID(csiVolID string) string { } // exportFilesystem - Method to export filesystem with idempotency -func (s *service) exportFilesystem(ctx context.Context, req *csi.ControllerPublishVolumeRequest, client *goscaleio.Client, fs *siotypes.FileSystem, nodeIP, nodeID string, am *csi.VolumeCapability_AccessMode) (*csi.ControllerPublishVolumeResponse, error) { - volumeContext := req.GetVolumeContext() - - // fetch the node IP, for exporting the FS on host. - // TODO - - // Create NFS export if it doesn't exist +func (s *service) exportFilesystem(ctx context.Context, req *csi.ControllerPublishVolumeRequest, client *goscaleio.Client, fs *siotypes.FileSystem, nodeIP, nodeID string, pContext map[string]string, am *csi.VolumeCapability_AccessMode) (*csi.ControllerPublishVolumeResponse, error) { nfsExportName := NFSExportNamePrefix + fs.Name nfsExportExists := true var nfsExportID string - + // Check if nfs export exists for the File system nfsExport, err := client.GetNFSExportByIDName(fs.ID, "") if err != nil { @@ -953,7 +947,7 @@ func (s *service) exportFilesystem(ctx context.Context, req *csi.ControllerPubli return nil, err } } - + // Create NFS export if it doesn't exist if !nfsExportExists { resp, err := client.CreateNFSExport(&siotypes.NFSExportCreate{ Name: nfsExportName, @@ -971,8 +965,6 @@ func (s *service) exportFilesystem(ctx context.Context, req *csi.ControllerPubli } - //Allocate host access to NFS Share with appropriate access mode - nfsExportResp, err := client.GetNFSExportByIDName(nfsExportID, "") if err != nil { @@ -1044,17 +1036,12 @@ func (s *service) exportFilesystem(ctx context.Context, req *csi.ControllerPubli return nil, status.Errorf(codes.NotFound, "Other hosts have access on NFS Share: %s", nfsExportID) } - // create publish context - publishContext := make(map[string]string) - publishContext[KeyNasName] = volumeContext[KeyNasName] - publishContext[KeyNfsACL] = volumeContext[KeyNfsACL] - //Idempotent case if foundIdempotent { Log.Info("Host has access to the given host and exists in the required state.") - return &csi.ControllerPublishVolumeResponse{PublishContext: volumeContext}, nil + return &csi.ControllerPublishVolumeResponse{PublishContext: pContext}, nil } - + //Allocate host access to NFS Share with appropriate access mode if am.Mode == csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY { readHostList = append(readHostList, nodeIP) client.ModifyNFSExport(&siotypes.NFSExportModify{AddReadOnlyRootHosts: readHostList}, nfsExportID) @@ -1067,12 +1054,9 @@ func (s *service) exportFilesystem(ctx context.Context, req *csi.ControllerPubli return nil, status.Errorf(codes.NotFound, "Allocating host %s access to NFS Export failed. Error: %v", nodeID, err) } Log.Debugf("NFS Export: %s is accessible to host: %s with access mode: %s", nfsExportID, nodeID, am.Mode) - Log.Debugf("ControllerPublishVolume successful for volid: [%s]", publishContext["volumeContextId"]) - // Add host IP to existing nfs export - // Add external host IP to existing nfs export - // TODO + Log.Debugf("ControllerPublishVolume successful for volid: [%s]", pContext["volumeContextId"]) - return &csi.ControllerPublishVolumeResponse{PublishContext: publishContext}, nil + return &csi.ControllerPublishVolumeResponse{PublishContext: pContext}, nil } // this function updates volumePrefixToSystems, a map of volume ID prefixes -> system IDs diff --git a/service/service_unit_test.go b/service/service_unit_test.go index 51b85021..66eea2e9 100644 --- a/service/service_unit_test.go +++ b/service/service_unit_test.go @@ -79,7 +79,7 @@ func TestGetVolSize(t *testing.T) { tt := tt t.Run("", func(st *testing.T) { st.Parallel() - size, err := validateVolSize(tt.cr, false) + size, err := validateVolSize(tt.cr) if tt.sizeKiB == 0 { // error is expected assert.Error(st, err) From a0e9f92b8271d4cfb4ee1404ab68883207c4d506 Mon Sep 17 00:00:00 2001 From: Khare Date: Thu, 11 May 2023 11:51:29 +0530 Subject: [PATCH 20/40] files updated --- service/controller.go | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/service/controller.go b/service/controller.go index 9b1106a3..0611e43f 100644 --- a/service/controller.go +++ b/service/controller.go @@ -236,6 +236,19 @@ func (s *service) CreateVolume( } } + // fetch volume name + name := req.GetName() + if name == "" { + return nil, status.Error(codes.InvalidArgument, + "Name cannot be empty") + } + + if len(name) > 31 { + name = name[0:31] + Log.Printf("Requested name %s longer than 31 character max, truncated to %s\n", req.Name, name) + req.Name = name + } + nfsAcls := s.opts.NfsAcls var arr *ArrayConnectionData sysID := s.opts.defaultSystemID @@ -281,10 +294,7 @@ func (s *service) CreateVolume( nfsAcls = arr.NfsAcls // Secrets next } - // fetch volume name - volName := req.GetName() - - // volume size + // fetch volume size size := cr.GetRequiredBytes() // log all parameters used in CreateVolume call @@ -377,18 +387,6 @@ func (s *service) CreateVolume( volType := s.getVolProvisionType(params) // Thick or Thin - name := req.GetName() - if name == "" { - return nil, status.Error(codes.InvalidArgument, - "Name cannot be empty") - } - - if len(name) > 31 { - name = name[0:31] - Log.Printf("Requested name %s longer than 31 character max, truncated to %s\n", req.Name, name) - req.Name = name - } - contentSource := req.GetVolumeContentSource() if contentSource != nil { volumeSource := contentSource.GetVolume() From 0bc1ba6dce6c1190135228c8848904f90635ecc5 Mon Sep 17 00:00:00 2001 From: Vamsikrishna_Siddu Date: Thu, 11 May 2023 07:35:40 -0400 Subject: [PATCH 21/40] removed debug logs. --- service/controller.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/service/controller.go b/service/controller.go index 000c0136..5d50e7ef 100644 --- a/service/controller.go +++ b/service/controller.go @@ -267,7 +267,7 @@ func (s *service) CreateVolume( return nil, status.Errorf(codes.InvalidArgument, "`%s` is a required parameter", KeyNasName) } nasServerID, err := s.getNASServerIDFromName(systemID, nasName) - fmt.Println("nasSeverID:", nasServerID) + if err != nil { return nil, err } @@ -282,7 +282,7 @@ func (s *service) CreateVolume( if err != nil { return nil, err } - fmt.Println("pdID:", pdID) + } storagePoolName, ok := params[KeyStoragePool] @@ -295,8 +295,6 @@ func (s *service) CreateVolume( return nil, err } - fmt.Println("storagePoolID:", storagePoolID) - // fetch NFS ACL if params[KeyNfsACL] != "" { nfsAcls = params[KeyNfsACL] // Storage class takes precedence @@ -326,8 +324,6 @@ func (s *service) CreateVolume( NasServerID: nasServerID, } - fmt.Printf("volumeParam:%#v\n", volumeParam) - //Idempotency check system, err := s.adminClients[systemID].FindSystem(systemID, "", "") if err != nil { @@ -354,7 +350,6 @@ func (s *service) CreateVolume( } Log.Debug("Volume does not exist, proceeding to create new volume") fsResp, err := system.CreateFileSystem(volumeParam) - fmt.Println("fsResp:", fsResp) if err != nil { Log.Debugf("Create volume response error:%v", err) return nil, status.Errorf(codes.Unknown, "Create Volume %s failed with error: %v", volName, err) @@ -895,7 +890,6 @@ func (s *service) ControllerPublishVolume( } if isNFS { fsID := getFilesystemIDFromCsiVolumeID(csiVolID) - fmt.Println("fsID:", fsID) fs, err := s.getFilesystemByID(fsID, systemID) if err != nil { if strings.EqualFold(err.Error(), sioGatewayFilesystemNotFound) || strings.Contains(err.Error(), "must be a hexadecimal number") { From f0049833bf9313ef59378e95456b46c8a6560683 Mon Sep 17 00:00:00 2001 From: Vamsikrishna_Siddu Date: Thu, 11 May 2023 07:39:04 -0400 Subject: [PATCH 22/40] remove debug logs. --- service/node.go | 1 - service/service.go | 2 -- 2 files changed, 3 deletions(-) diff --git a/service/node.go b/service/node.go index 5d4e1c7c..2167e823 100644 --- a/service/node.go +++ b/service/node.go @@ -650,7 +650,6 @@ func (s *service) NodeGetInfo( // Create the topology keys // csi-vxflexos.dellemc.com/: topology := map[string]string{} - fmt.Printf("%#v", connectedSystemID) for _, sysID := range connectedSystemID { isNFS, err := s.checkNFS(ctx, sysID) if err != nil { diff --git a/service/service.go b/service/service.go index dcc087c3..f115bbf5 100644 --- a/service/service.go +++ b/service/service.go @@ -469,8 +469,6 @@ func (s *service) checkNFS(ctx context.Context, systemID string) (bool, error) { } c := s.adminClients[systemID] - fmt.Printf("adminClients:%#v\n", s.adminClients) - fmt.Printf("client: %#v\n", c) if c == nil { return false, nil From 331a1c033cb5b210ed2becd419301b179a24bd0c Mon Sep 17 00:00:00 2001 From: Vamsikrishna_Siddu Date: Thu, 11 May 2023 11:06:52 -0400 Subject: [PATCH 23/40] add debug. --- service/controller.go | 1 + service/service.go | 1 + 2 files changed, 2 insertions(+) diff --git a/service/controller.go b/service/controller.go index 49a4de4e..2e461ce6 100644 --- a/service/controller.go +++ b/service/controller.go @@ -259,6 +259,7 @@ func (s *service) CreateVolume( sysID := s.opts.defaultSystemID arr = s.opts.arrays[sysID] volName := name + volName = "k8s-9cbf02ca15" if isNFS { // fetch NAS server ID diff --git a/service/service.go b/service/service.go index 1027d7c3..b1b4147c 100644 --- a/service/service.go +++ b/service/service.go @@ -946,6 +946,7 @@ func (s *service) exportFilesystem(ctx context.Context, req *csi.ControllerPubli } // Create NFS export if it doesn't exist if !nfsExportExists { + Log.Debugf("NFS Export does not exist for fs: %s ,proceeding to create NFS Export") resp, err := client.CreateNFSExport(&siotypes.NFSExportCreate{ Name: nfsExportName, FileSystemID: fs.ID, From 3e3ea6caa73458dcf0d29b986761e334fcf0468c Mon Sep 17 00:00:00 2001 From: Vamsikrishna_Siddu Date: Thu, 11 May 2023 12:13:57 -0400 Subject: [PATCH 24/40] add debug. --- service/controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/controller.go b/service/controller.go index 2e461ce6..118f566e 100644 --- a/service/controller.go +++ b/service/controller.go @@ -259,7 +259,7 @@ func (s *service) CreateVolume( sysID := s.opts.defaultSystemID arr = s.opts.arrays[sysID] volName := name - volName = "k8s-9cbf02ca15" + volName = "tweeFS3" if isNFS { // fetch NAS server ID From c7ce0f59f804745d047432282731970858dabd0d Mon Sep 17 00:00:00 2001 From: Vamsikrishna_Siddu Date: Thu, 11 May 2023 12:35:58 -0400 Subject: [PATCH 25/40] add debug3. --- service/controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/controller.go b/service/controller.go index 118f566e..2e461ce6 100644 --- a/service/controller.go +++ b/service/controller.go @@ -259,7 +259,7 @@ func (s *service) CreateVolume( sysID := s.opts.defaultSystemID arr = s.opts.arrays[sysID] volName := name - volName = "tweeFS3" + volName = "k8s-9cbf02ca15" if isNFS { // fetch NAS server ID From 1ebf863b9c8e65335e26b56648341f82ba10f2fc Mon Sep 17 00:00:00 2001 From: Vamsikrishna_Siddu Date: Thu, 11 May 2023 13:42:43 -0400 Subject: [PATCH 26/40] fixed a mistake. --- service/service.go | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/service/service.go b/service/service.go index b1b4147c..d715b8da 100644 --- a/service/service.go +++ b/service/service.go @@ -934,16 +934,24 @@ func (s *service) exportFilesystem(ctx context.Context, req *csi.ControllerPubli nfsExportExists := true var nfsExportID string // Check if nfs export exists for the File system - nfsExport, err := client.GetNFSExportByIDName(fs.ID, "") + if fs.ExportFsID != "" { + nfsExport, err := client.GetNFSExportByIDName(fs.ExportFsID, "") - if err != nil { - if strings.Contains(err.Error(), sioGatewayNFSExportNotFound) { - nfsExportExists = false + if err != nil { + if strings.Contains(err.Error(), sioGatewayNFSExportNotFound) { + nfsExportExists = false + } else { + return nil, err + } } else { - return nil, err + nfsExportID = nfsExport.ID } + + } else { + nfsExportExists = false } + // Create NFS export if it doesn't exist if !nfsExportExists { Log.Debugf("NFS Export does not exist for fs: %s ,proceeding to create NFS Export") @@ -958,9 +966,6 @@ func (s *service) exportFilesystem(ctx context.Context, req *csi.ControllerPubli } nfsExportID = resp.ID - } else { - nfsExportID = nfsExport.ID - } nfsExportResp, err := client.GetNFSExportByIDName(nfsExportID, "") From 850e7724d2069447fd93fff95006dd186022efc6 Mon Sep 17 00:00:00 2001 From: Vamsikrishna_Siddu Date: Thu, 11 May 2023 14:12:27 -0400 Subject: [PATCH 27/40] add debug. --- service/service.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/service/service.go b/service/service.go index d715b8da..6d82202b 100644 --- a/service/service.go +++ b/service/service.go @@ -931,6 +931,9 @@ func (s *service) exportFilesystem(ctx context.Context, req *csi.ControllerPubli nfsExportName := NFSExportNamePrefix + fs.Name + fmt.Printf("fileSystem:%#v\n", fs) + fmt.Printf("exportID:%#v\n", fs.ExportFsID) + nfsExportExists := true var nfsExportID string // Check if nfs export exists for the File system @@ -955,6 +958,9 @@ func (s *service) exportFilesystem(ctx context.Context, req *csi.ControllerPubli // Create NFS export if it doesn't exist if !nfsExportExists { Log.Debugf("NFS Export does not exist for fs: %s ,proceeding to create NFS Export") + fmt.Printf("nfsExportName:%v\n", nfsExportName) + fmt.Printf("filesytemID:%v\n", fs.ID) + fmt.Println("path",NFSExportLocalPath + fs.Name) resp, err := client.CreateNFSExport(&siotypes.NFSExportCreate{ Name: nfsExportName, FileSystemID: fs.ID, From ffd1a2ae4e3cf29a08215c478f8d812e03f53cf9 Mon Sep 17 00:00:00 2001 From: Vamsikrishna_Siddu Date: Fri, 12 May 2023 09:55:41 +0530 Subject: [PATCH 28/40] fixed th idempotency. --- service/service.go | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/service/service.go b/service/service.go index 6d82202b..7143f71e 100644 --- a/service/service.go +++ b/service/service.go @@ -929,30 +929,28 @@ func (s *service) getSystemIDFromCsiVolumeID(csiVolID string) string { // exportFilesystem - Method to export filesystem with idempotency func (s *service) exportFilesystem(ctx context.Context, req *csi.ControllerPublishVolumeRequest, client *goscaleio.Client, fs *siotypes.FileSystem, nodeIP, nodeID string, pContext map[string]string, am *csi.VolumeCapability_AccessMode) (*csi.ControllerPublishVolumeResponse, error) { - nfsExportName := NFSExportNamePrefix + fs.Name + var nfsExportName string + nfsExportName = NFSExportNamePrefix + fs.Name fmt.Printf("fileSystem:%#v\n", fs) fmt.Printf("exportID:%#v\n", fs.ExportFsID) - nfsExportExists := true + nfsExportExists := false var nfsExportID string + // Check if nfs export exists for the File system - if fs.ExportFsID != "" { - nfsExport, err := client.GetNFSExportByIDName(fs.ExportFsID, "") + nfsExportList, err := client.GetNFSExport() - if err != nil { - if strings.Contains(err.Error(), sioGatewayNFSExportNotFound) { - nfsExportExists = false + if err != nil { + return nil, err + } - } else { - return nil, err - } - } else { + for _, nfsExport := range nfsExportList { + if nfsExport.FileSystemID == fs.ID { + nfsExportExists = true nfsExportID = nfsExport.ID + nfsExportName = nfsExport.Name } - - } else { - nfsExportExists = false } // Create NFS export if it doesn't exist @@ -960,7 +958,7 @@ func (s *service) exportFilesystem(ctx context.Context, req *csi.ControllerPubli Log.Debugf("NFS Export does not exist for fs: %s ,proceeding to create NFS Export") fmt.Printf("nfsExportName:%v\n", nfsExportName) fmt.Printf("filesytemID:%v\n", fs.ID) - fmt.Println("path",NFSExportLocalPath + fs.Name) + fmt.Println("path", NFSExportLocalPath+fs.Name) resp, err := client.CreateNFSExport(&siotypes.NFSExportCreate{ Name: nfsExportName, FileSystemID: fs.ID, From 1182759fd7351396cbfca8e4942f992273789a3e Mon Sep 17 00:00:00 2001 From: Vamsikrishna_Siddu Date: Fri, 12 May 2023 10:37:26 +0530 Subject: [PATCH 29/40] remove debug logs. --- service/service.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/service/service.go b/service/service.go index 7143f71e..1f66fa1f 100644 --- a/service/service.go +++ b/service/service.go @@ -932,9 +932,6 @@ func (s *service) exportFilesystem(ctx context.Context, req *csi.ControllerPubli var nfsExportName string nfsExportName = NFSExportNamePrefix + fs.Name - fmt.Printf("fileSystem:%#v\n", fs) - fmt.Printf("exportID:%#v\n", fs.ExportFsID) - nfsExportExists := false var nfsExportID string @@ -955,10 +952,7 @@ func (s *service) exportFilesystem(ctx context.Context, req *csi.ControllerPubli // Create NFS export if it doesn't exist if !nfsExportExists { - Log.Debugf("NFS Export does not exist for fs: %s ,proceeding to create NFS Export") - fmt.Printf("nfsExportName:%v\n", nfsExportName) - fmt.Printf("filesytemID:%v\n", fs.ID) - fmt.Println("path", NFSExportLocalPath+fs.Name) + Log.Debugf("NFS Export does not exist for fs: %s ,proceeding to create NFS Export", fs) resp, err := client.CreateNFSExport(&siotypes.NFSExportCreate{ Name: nfsExportName, FileSystemID: fs.ID, From 1d65ccd040b6bc7c90fe1ab0edff6b2c79bb4e10 Mon Sep 17 00:00:00 2001 From: Vamsikrishna_Siddu Date: Fri, 12 May 2023 11:02:20 +0530 Subject: [PATCH 30/40] removed the hardcoded name. --- service/controller.go | 1 - 1 file changed, 1 deletion(-) diff --git a/service/controller.go b/service/controller.go index 2e461ce6..49a4de4e 100644 --- a/service/controller.go +++ b/service/controller.go @@ -259,7 +259,6 @@ func (s *service) CreateVolume( sysID := s.opts.defaultSystemID arr = s.opts.arrays[sysID] volName := name - volName = "k8s-9cbf02ca15" if isNFS { // fetch NAS server ID From 0c2974037e989e186905d7114de51befea8773bb Mon Sep 17 00:00:00 2001 From: Vamsikrishna_Siddu Date: Sat, 13 May 2023 11:53:33 +0530 Subject: [PATCH 31/40] node publish volume nfs changes. --- service/mount.go | 112 +++++++++++++++++++++++++++++++++++++++++++++ service/node.go | 32 +++++++++++++ service/service.go | 18 ++++++++ 3 files changed, 162 insertions(+) diff --git a/service/mount.go b/service/mount.go index 313c4687..7d6a9329 100644 --- a/service/mount.go +++ b/service/mount.go @@ -331,6 +331,118 @@ func publishVolume( return nil } +func publishNFS(ctx context.Context, req *csi.NodePublishVolumeRequest, nfsExportUrl string) error { + volCap := req.GetVolumeCapability() + + if volCap == nil { + return status.Error(codes.InvalidArgument, + "Volume Capability is required") + } + + am := volCap.GetAccessMode() + + if am == nil { + return status.Error(codes.InvalidArgument, + "Volume Access Mode is required") + } + + mountVol := volCap.GetMount() + + if mountVol == nil { + return status.Error(codes.InvalidArgument, "Invalid access type") + } + + var mntOptions []string + mntOptions = mountVol.GetMountFlags() + Log.Infof("The mountOptions received are: %s", mntOptions) + + target := req.GetTargetPath() + if target == "" { + return status.Error(codes.InvalidArgument, + "Target Path is required") + } + + // make sure target is created + _, err := mkdir(target) + + if err != nil { + return status.Error(codes.FailedPrecondition, fmt.Sprintf("Could not create '%s': '%s'", target, err.Error())) + } + roFlag := req.GetReadonly() + rwOption := "rw" + if roFlag { + rwOption = "ro" + } + + mntOptions = append(mntOptions, rwOption) + + fields := map[string]interface{}{ + "ID": req.VolumeId, + "TargetPath": target, + "ExportPath": nfsExportUrl, + "AccessMode": am.GetMode(), + } + Log.WithFields(fields).Info("Node publish volume params ") + + mnts, err := gofsutil.GetMounts(ctx) + if err != nil { + return status.Errorf(codes.Internal, + "could not reliably determine existing mount status: '%s'", + err.Error()) + } + + if len(mnts) != 0 { + for _, m := range mnts { + // check for idempotency + //same volume + if m.Device == nfsExportUrl { + if m.Path == target { + //as per specs, T1=T2, P1=P2 - return OK + if contains(m.Opts, rwOption) { + Log.WithFields(fields).Debug( + "mount already in place with same options") + return nil + } + //T1=T2, P1!=P2 - return AlreadyExists + Log.WithFields(fields).Error("Mount point already in use by device with different options") + return status.Error(codes.AlreadyExists, "Mount point already in use by device with different options") + } + //T1!=T2, P1==P2 || P1 != P2 - return FailedPrecondition for single node + if am.GetMode() == csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER || + am.GetMode() == csi.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY || + am.GetMode() == csi.VolumeCapability_AccessMode_SINGLE_NODE_SINGLE_WRITER { + Log.WithFields(fields).Error("Mount point already in use for same device") + return status.Error(codes.FailedPrecondition, "Mount point already in use for same device") + } + } + } + } + + Log.Infof("The mountOptions being used for mount are: %s", mntOptions) + if err := gofsutil.Mount(context.Background(), nfsExportUrl, target, "nfs", mntOptions...); err != nil { + var count = 0 + var errmsg = err.Error() + //Both substring validation is for NFSv3 and NFSv4 errors resp. + for (strings.Contains(strings.ToLower(errmsg), "access denied by server while mounting") || (strings.Contains(strings.ToLower(errmsg), "no such file or directory"))) && count < 5 { + time.Sleep(2 * time.Second) + Log.Infof("Mount re-trial attempt-%d", count) + err = gofsutil.Mount(context.Background(), nfsExportUrl, target, "nfs", mntOptions...) + if err != nil { + errmsg = err.Error() + } else { + break + } + count++ + } + if err != nil { + Log.Errorf("%v", err) + return err + } + } + return nil + +} + func handlePrivFSMount( ctx context.Context, accMode *csi.VolumeCapability_AccessMode, diff --git a/service/node.go b/service/node.go index 2167e823..4d9cb524 100644 --- a/service/node.go +++ b/service/node.go @@ -142,6 +142,13 @@ func (s *service) NodePublishVolume( } Log.Printf("[NodePublishVolume] csiVolID: %s", csiVolID) + // Check for NFS protocol + fsType := volumeContext[KeyFsType] + isNFS := false + if fsType == "nfs" { + isNFS = true + } + volID := getVolumeIDFromCsiVolumeID(csiVolID) Log.Printf("[NodePublishVolume] volumeID: %s", volID) @@ -170,6 +177,31 @@ func (s *service) NodePublishVolume( } + if isNFS { + fsID := getFilesystemIDFromCsiVolumeID(csiVolID) + + fs, err := s.getFilesystemByID(fsID, systemID) + if err != nil { + if strings.EqualFold(err.Error(), sioGatewayFilesystemNotFound) || strings.Contains(err.Error(), "must be a hexadecimal number") { + return nil, status.Error(codes.NotFound, + "filesystem not found") + } + return nil, status.Errorf(codes.Internal, + "failure checking filesystem status before controller publish: %s", + err.Error()) + } + + client := s.adminClients[systemID] + + NFSExport, err := s.getNFSExport(fs, client) + + if err != nil { + return nil, err + } + + publishNFS(ctx, req, NFSExport.Path) + } + sdcMappedVol, err := s.getSDCMappedVol(volID, systemID, publishGetMappedVolMaxRetry) if err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) diff --git a/service/service.go b/service/service.go index 1f66fa1f..f8f8fb37 100644 --- a/service/service.go +++ b/service/service.go @@ -892,6 +892,24 @@ func getFilesystemIDFromCsiVolumeID(csiVolID string) string { return "" } +func (s *service) getNFSExport(fs *siotypes.FileSystem, client *goscaleio.Client) (*siotypes.NFSExport, error) { + + nfsExportList, err := client.GetNFSExport() + + if err != nil { + return nil, err + } + + for _, nfsExport := range nfsExportList { + if nfsExport.FileSystemID == fs.ID { + return &nfsExport, nil + } + } + + return nil, status.Errorf(codes.NotFound, "NFS Export for the file system: %s not found", fs.Name) + +} + // getSystemIDFromCsiVolumeId returns PowerFlex volume ID from CSI volume ID func (s *service) getSystemIDFromCsiVolumeID(csiVolID string) string { containsHyphen := strings.Contains(csiVolID, "/") From a26665b54ed301880e2a5b27b0e4231dbf91176a Mon Sep 17 00:00:00 2001 From: Vamsikrishna_Siddu Date: Sat, 13 May 2023 12:32:57 +0530 Subject: [PATCH 32/40] modified the dockerfile. --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 179dc22b..293851a4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,6 +35,7 @@ RUN microdnf update -y && \ kmod \ libaio \ numactl \ + nfs-utils \ xfsprogs && \ microdnf clean all ENTRYPOINT ["/csi-vxflexos.sh"] From ec50237294acab64b602caf69834bab7fef5cd88 Mon Sep 17 00:00:00 2001 From: Vamsikrishna_Siddu Date: Sat, 13 May 2023 13:36:59 +0530 Subject: [PATCH 33/40] fixed the path to send to nfs mount. --- service/node.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/service/node.go b/service/node.go index 4d9cb524..a566a877 100644 --- a/service/node.go +++ b/service/node.go @@ -178,6 +178,9 @@ func (s *service) NodePublishVolume( } if isNFS { + pubContext := req.PublishContext + host := pubContext["host"] + fmt.Printf("host:%#v\n", host) fsID := getFilesystemIDFromCsiVolumeID(csiVolID) fs, err := s.getFilesystemByID(fsID, systemID) @@ -199,7 +202,11 @@ func (s *service) NodePublishVolume( return nil, err } - publishNFS(ctx, req, NFSExport.Path) + path := fmt.Sprintf("%s:%s", host, NFSExport.Path) + + fmt.Printf("path:%#v\n", path) + + publishNFS(ctx, req, path) } sdcMappedVol, err := s.getSDCMappedVol(volID, systemID, publishGetMappedVolMaxRetry) From 6d1eb2670e76d411bd71c0dacea5281ae897b01e Mon Sep 17 00:00:00 2001 From: Vamsikrishna_Siddu Date: Sat, 13 May 2023 15:13:36 +0530 Subject: [PATCH 34/40] added fileinterface functionality. --- go.mod | 2 +- go.sum | 2 ++ service/node.go | 20 +++++++++++++++++++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 6dd31381..83e8c16e 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/dell/dell-csi-extensions/volumeGroupSnapshot v1.2.2 github.com/dell/gocsi v1.7.0 github.com/dell/gofsutil v1.12.0 - github.com/dell/goscaleio v1.10.1-0.20230502150156-f467d3984623 + github.com/dell/goscaleio v1.10.1-0.20230508114247-87a9476ec860 github.com/fsnotify/fsnotify v1.5.1 github.com/golang/protobuf v1.5.3 github.com/google/uuid v1.3.0 diff --git a/go.sum b/go.sum index d93020fe..2c3f191c 100644 --- a/go.sum +++ b/go.sum @@ -116,6 +116,8 @@ github.com/dell/gofsutil v1.12.0 h1:oo2YHfGFKHvHS1urtqjOIKpaHwcdyqacwKHLXzUg33M= github.com/dell/gofsutil v1.12.0/go.mod h1:mGMN5grVDtHv2imNw5+gFr2RmCqeyYgBFBldUbHtV78= github.com/dell/goscaleio v1.10.1-0.20230502150156-f467d3984623 h1:PQwO9B6aI5IF1NI9KjxRZCFm5Mk2YkB21b9SEcytmgI= github.com/dell/goscaleio v1.10.1-0.20230502150156-f467d3984623/go.mod h1:dMTrHnXSsPus+Kd9mrs0JuyrCndoKvFP/bbEdc21Bi8= +github.com/dell/goscaleio v1.10.1-0.20230508114247-87a9476ec860 h1:0+H9wozXtaY8esJRDEdMWoTppTUJYvHfYjbilAGHsU0= +github.com/dell/goscaleio v1.10.1-0.20230508114247-87a9476ec860/go.mod h1:dMTrHnXSsPus+Kd9mrs0JuyrCndoKvFP/bbEdc21Bi8= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= diff --git a/service/node.go b/service/node.go index a566a877..c874b84e 100644 --- a/service/node.go +++ b/service/node.go @@ -202,7 +202,25 @@ func (s *service) NodePublishVolume( return nil, err } - path := fmt.Sprintf("%s:%s", host, NFSExport.Path) + system, err := client.FindSystem(systemID, "", "") + + if err != nil { + return nil, err + } + + nas, err := system.GetNASByIDName(fs.NasServerID, "") + + if err != nil { + return nil, err + } + + fileInterface, err := system.GetFileInterface(nas.CurrentPreferredIPv4InterfaceID) + + if err != nil { + return nil, err + } + + path := fmt.Sprintf("%s:%s", fileInterface.IpAddress, NFSExport.Path) fmt.Printf("path:%#v\n", path) From 7f5b27d9a15d4f4f28fc2194aa2ac8208f90e8d3 Mon Sep 17 00:00:00 2001 From: Vamsikrishna_Siddu Date: Sat, 13 May 2023 15:37:15 +0530 Subject: [PATCH 35/40] fixed a small mistake. --- service/node.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/service/node.go b/service/node.go index c874b84e..edfd7c68 100644 --- a/service/node.go +++ b/service/node.go @@ -224,7 +224,11 @@ func (s *service) NodePublishVolume( fmt.Printf("path:%#v\n", path) - publishNFS(ctx, req, path) + if err := publishNFS(ctx, req, path); err != nil { + return nil, err + } + + return &csi.NodePublishVolumeResponse{}, nil } sdcMappedVol, err := s.getSDCMappedVol(volID, systemID, publishGetMappedVolMaxRetry) From eaedc7e7305505c441dfe6a4585b0f9b864df974 Mon Sep 17 00:00:00 2001 From: Vamsikrishna_Siddu Date: Mon, 15 May 2023 15:34:32 +0530 Subject: [PATCH 36/40] update controller.go --- service/controller.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/service/controller.go b/service/controller.go index 49a4de4e..3a33f7c8 100644 --- a/service/controller.go +++ b/service/controller.go @@ -98,8 +98,6 @@ const ( removeModeOnlyMe = "ONLY_ME" sioGatewayNotFound = "Not found" sioGatewayVolumeNotFound = "Could not find the volume" - sioGatewayNFSExportNotFound = "couldn't find NFS export" - sioGatewayFilesystemNotFound = "Could not find the filesystem" sioVolumeRemovalOperationInProgress = "A volume removal operation is currently in progress" sioGatewayVolumeNameInUse = "Volume name already in use. Please use a different name." errNoMultiMap = "volume not enabled for mapping to multiple hosts" @@ -891,12 +889,12 @@ func (s *service) ControllerPublishVolume( fsID := getFilesystemIDFromCsiVolumeID(csiVolID) fs, err := s.getFilesystemByID(fsID, systemID) if err != nil { - if strings.EqualFold(err.Error(), sioGatewayFilesystemNotFound) || strings.Contains(err.Error(), "must be a hexadecimal number") { + if strings.EqualFold(err.Error(), sioGatewayVolumeNotFound) || strings.Contains(err.Error(), "must be a hexadecimal number") { return nil, status.Error(codes.NotFound, - "filesystem not found") + "volume not found") } return nil, status.Errorf(codes.Internal, - "failure checking filesystem status before controller publish: %s", + "failure checking volume status before controller publish: %s", err.Error()) } From aeb733b4d6c0ce333899581bf2eabe6691e831f7 Mon Sep 17 00:00:00 2001 From: Vamsikrishna_Siddu Date: Mon, 15 May 2023 15:54:53 +0530 Subject: [PATCH 37/40] files updated. --- service/node.go | 22 ++-------------------- service/service.go | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/service/node.go b/service/node.go index edfd7c68..335b89aa 100644 --- a/service/node.go +++ b/service/node.go @@ -178,14 +178,11 @@ func (s *service) NodePublishVolume( } if isNFS { - pubContext := req.PublishContext - host := pubContext["host"] - fmt.Printf("host:%#v\n", host) fsID := getFilesystemIDFromCsiVolumeID(csiVolID) fs, err := s.getFilesystemByID(fsID, systemID) if err != nil { - if strings.EqualFold(err.Error(), sioGatewayFilesystemNotFound) || strings.Contains(err.Error(), "must be a hexadecimal number") { + if strings.EqualFold(err.Error(), sioGatewayVolumeNotFound) || strings.Contains(err.Error(), "must be a hexadecimal number") { return nil, status.Error(codes.NotFound, "filesystem not found") } @@ -202,28 +199,13 @@ func (s *service) NodePublishVolume( return nil, err } - system, err := client.FindSystem(systemID, "", "") - - if err != nil { - return nil, err - } - - nas, err := system.GetNASByIDName(fs.NasServerID, "") - - if err != nil { - return nil, err - } - - fileInterface, err := system.GetFileInterface(nas.CurrentPreferredIPv4InterfaceID) - + fileInterface, err := s.getFileInterface(systemID, fs, client) if err != nil { return nil, err } path := fmt.Sprintf("%s:%s", fileInterface.IpAddress, NFSExport.Path) - fmt.Printf("path:%#v\n", path) - if err := publishNFS(ctx, req, path); err != nil { return nil, err } diff --git a/service/service.go b/service/service.go index f8f8fb37..172d24e4 100644 --- a/service/service.go +++ b/service/service.go @@ -910,6 +910,27 @@ func (s *service) getNFSExport(fs *siotypes.FileSystem, client *goscaleio.Client } +func (s *service) getFileInterface(systemID string, fs *siotypes.FileSystem, client *goscaleio.Client) (*siotypes.FileInterface, error) { + system, err := client.FindSystem(systemID, "", "") + + if err != nil { + return nil, err + } + + nas, err := system.GetNASByIDName(fs.NasServerID, "") + + if err != nil { + return nil, err + } + + fileInterface, err := system.GetFileInterface(nas.CurrentPreferredIPv4InterfaceID) + + if err != nil { + return nil, err + } + return fileInterface, err +} + // getSystemIDFromCsiVolumeId returns PowerFlex volume ID from CSI volume ID func (s *service) getSystemIDFromCsiVolumeID(csiVolID string) string { containsHyphen := strings.Contains(csiVolID, "/") From 5d91e314f47a1bf73f748e5989d4dd2e4c1d391e Mon Sep 17 00:00:00 2001 From: Vamsikrishna_Siddu Date: Mon, 15 May 2023 16:18:17 +0530 Subject: [PATCH 38/40] updated nfs storageclass. --- samples/storageclass/storageclass-nfs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/storageclass/storageclass-nfs.yaml b/samples/storageclass/storageclass-nfs.yaml index a89568ef..3a3a2ceb 100644 --- a/samples/storageclass/storageclass-nfs.yaml +++ b/samples/storageclass/storageclass-nfs.yaml @@ -94,4 +94,4 @@ allowedTopologies: - matchLabelExpressions: - key: csi-vxflexos.dellemc.com/-nfs # Insert System ID values: - - csi-vxflexos.dellemc.com + - "true" From 0201314c2a164fc4ed7fe6bbd16327f41091a495 Mon Sep 17 00:00:00 2001 From: Vamsikrishna_Siddu Date: Mon, 15 May 2023 16:58:01 +0530 Subject: [PATCH 39/40] fixed the debug log. --- service/service.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/service.go b/service/service.go index 172d24e4..49025ce0 100644 --- a/service/service.go +++ b/service/service.go @@ -991,7 +991,7 @@ func (s *service) exportFilesystem(ctx context.Context, req *csi.ControllerPubli // Create NFS export if it doesn't exist if !nfsExportExists { - Log.Debugf("NFS Export does not exist for fs: %s ,proceeding to create NFS Export", fs) + Log.Debugf("NFS Export does not exist for fs: %s ,proceeding to create NFS Export", fs.Name) resp, err := client.CreateNFSExport(&siotypes.NFSExportCreate{ Name: nfsExportName, FileSystemID: fs.ID, From b1379f05ab68ef96c2bbee596d40ffc8770a96f9 Mon Sep 17 00:00:00 2001 From: Vamsikrishna_Siddu Date: Mon, 15 May 2023 17:14:58 +0530 Subject: [PATCH 40/40] updated files. --- go.mod | 2 +- go.sum | 6 ++---- service/node.go | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 83e8c16e..1735da25 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/dell/dell-csi-extensions/volumeGroupSnapshot v1.2.2 github.com/dell/gocsi v1.7.0 github.com/dell/gofsutil v1.12.0 - github.com/dell/goscaleio v1.10.1-0.20230508114247-87a9476ec860 + github.com/dell/goscaleio v1.10.1-0.20230515095359-d145e54cb0fe github.com/fsnotify/fsnotify v1.5.1 github.com/golang/protobuf v1.5.3 github.com/google/uuid v1.3.0 diff --git a/go.sum b/go.sum index 2c3f191c..7b7bc971 100644 --- a/go.sum +++ b/go.sum @@ -114,10 +114,8 @@ github.com/dell/gocsi v1.7.0 h1:fMQO2zwAXCaIsUoPCcnnuPMwfQMoaI1/0aqkQVndlxU= github.com/dell/gocsi v1.7.0/go.mod h1:X/8Ll8qqKAKCenmd1gPJMUvUmgY8cK0LiS8Pck12UaU= github.com/dell/gofsutil v1.12.0 h1:oo2YHfGFKHvHS1urtqjOIKpaHwcdyqacwKHLXzUg33M= github.com/dell/gofsutil v1.12.0/go.mod h1:mGMN5grVDtHv2imNw5+gFr2RmCqeyYgBFBldUbHtV78= -github.com/dell/goscaleio v1.10.1-0.20230502150156-f467d3984623 h1:PQwO9B6aI5IF1NI9KjxRZCFm5Mk2YkB21b9SEcytmgI= -github.com/dell/goscaleio v1.10.1-0.20230502150156-f467d3984623/go.mod h1:dMTrHnXSsPus+Kd9mrs0JuyrCndoKvFP/bbEdc21Bi8= -github.com/dell/goscaleio v1.10.1-0.20230508114247-87a9476ec860 h1:0+H9wozXtaY8esJRDEdMWoTppTUJYvHfYjbilAGHsU0= -github.com/dell/goscaleio v1.10.1-0.20230508114247-87a9476ec860/go.mod h1:dMTrHnXSsPus+Kd9mrs0JuyrCndoKvFP/bbEdc21Bi8= +github.com/dell/goscaleio v1.10.1-0.20230515095359-d145e54cb0fe h1:xG7XkoRorM/I9RhbQrFxDOdy5m9TLfYAZhaAX9vH138= +github.com/dell/goscaleio v1.10.1-0.20230515095359-d145e54cb0fe/go.mod h1:dMTrHnXSsPus+Kd9mrs0JuyrCndoKvFP/bbEdc21Bi8= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= diff --git a/service/node.go b/service/node.go index 335b89aa..afe5a0f2 100644 --- a/service/node.go +++ b/service/node.go @@ -204,7 +204,7 @@ func (s *service) NodePublishVolume( return nil, err } - path := fmt.Sprintf("%s:%s", fileInterface.IpAddress, NFSExport.Path) + path := fmt.Sprintf("%s:%s", fileInterface.IPAddress, NFSExport.Path) if err := publishNFS(ctx, req, path); err != nil { return nil, err