hpe-storage · datamattsson · May 6, 2023 · May 6, 2023
@@ -1,3 +1,4 @@
 e2e/e2e.test
 e2e/ginkgo
-e2e/reports/
+e2e/reports-*
+TODO
@@ -6,4 +6,4 @@ ADD requirements.txt /
 RUN pip3 install -r requirements.txt
 ADD truenascsp/*.py /app/
 WORKDIR /app
-ENTRYPOINT [ "gunicorn", "-b", "0.0.0.0:8080", "csp:SERVE" ]
+ENTRYPOINT [ "gunicorn", "--workers", "3", "--bind", "0.0.0.0:8080", "--timeout", "60", "csp:SERVE" ]
@@ -4,12 +4,11 @@ These procedures assumes a running Kubernetes cluster [supported by the HPE CSI
 
 ## Prerequisites
 
-- HPE CSI Driver for Kubernetes version 2.2.0 or later
-- TrueNAS 12.0 BETA or later
-- TrueNAS SCALE 22.02 RC1 or later
+- TrueNAS 12.0 or later
+- TrueNAS SCALE 22.02 or later
 - FreeNAS 11.2-U3 or later
-- Kubernetes 1.20 or later
-- Helm 3.6 or later (optional, only needed if using Helm to install the CSP)
+- Kubernetes 1.23 or later
+- Helm 3.6 or later (recommended, only needed if using Helm to install the CSP)
 
 ### TrueNAS Container Storage Provider Helm Chart
 
@@ -26,15 +25,15 @@ The HPE CSI Driver may be installed using either a Helm Chart, Operator or direc
 Install HPE CSI Driver using manifests:
 
 ```
-kubectl create -f https://raw.githubusercontent.com/hpe-storage/co-deployments/master/yaml/csi-driver/v2.2.0/hpe-linux-config.yaml
-kubectl create -f https://raw.githubusercontent.com/hpe-storage/co-deployments/master/yaml/csi-driver/v2.2.0/hpe-csi-k8s-1.22.yaml
+kubectl create -f https://raw.githubusercontent.com/hpe-storage/co-deployments/master/yaml/csi-driver/v2.3.0/hpe-linux-config.yaml
+kubectl create -f https://raw.githubusercontent.com/hpe-storage/co-deployments/master/yaml/csi-driver/v2.3.0/hpe-csi-k8s-1.26.yaml
 ```
 
 Install the TrueNAS CSP using manifests:
 
 ```
 kubectl create ns hpe-storage
-kubectl create -f https://raw.githubusercontent.com/hpe-storage/truenas-csp/master/K8s/v2.2.0/truenas-csp.yaml
+kubectl create -f https://raw.githubusercontent.com/hpe-storage/truenas-csp/master/K8s/v2.3.0/truenas-csp.yaml
 ```
 
 **Note:** Replace `hpe-csi-k8s-<version>.yaml` with your version of Kubernetes. Also change the version of the HPE CSI Driver manifests where applicable. Using mismatching versions of the TrueNAS CSP and the HPE CSI Driver will most likely **NOT** work.
@@ -73,7 +72,7 @@ The Target Global Configuration needs to be updated with this Base Name:
 
 ![](https://hpe-storage.github.io/truenas-csp/assets/global-target.png)
 
-- Base Name: `iqn.2011-08.org.truenas.ctl`
+- Base Name: `iqn.2011-08.org.truenas.ctl` or `iqn.2005-10.org.freenas.ctl`
 
 **Hint:** If TrueNAS/FreeNAS is not giving you the option to select nothing but 0.0.0.0 in the portal configuration is because you're using DHCP. Only statically assigned addresses can be used in the picker.
 

@@ -0,0 +1,25 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "false"
+  name: hpe-standard-nfs
+provisioner: csi.hpe.com
+parameters:
+  csi.storage.k8s.io/controller-expand-secret-name: truenas-secret
+  csi.storage.k8s.io/controller-expand-secret-namespace: hpe-storage
+  csi.storage.k8s.io/controller-publish-secret-name: truenas-secret
+  csi.storage.k8s.io/controller-publish-secret-namespace: hpe-storage
+  csi.storage.k8s.io/node-publish-secret-name: truenas-secret
+  csi.storage.k8s.io/node-publish-secret-namespace: hpe-storage
+  csi.storage.k8s.io/node-stage-secret-name: truenas-secret
+  csi.storage.k8s.io/node-stage-secret-namespace: hpe-storage
+  csi.storage.k8s.io/provisioner-secret-name: truenas-secret
+  csi.storage.k8s.io/provisioner-secret-namespace: hpe-storage
+  csi.storage.k8s.io/fstype: xfs
+  nfsResources: "true"
+  allowOverrides: sparse,compression,deduplication,volblocksize,sync,description  
+  root: zwimming/csi-volumes
+reclaimPolicy: Delete
+allowVolumeExpansion: true
@@ -0,0 +1,24 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "true"
+  name: hpe-standard
+provisioner: csi.hpe.com
+parameters:
+  csi.storage.k8s.io/controller-expand-secret-name: truenas-secret
+  csi.storage.k8s.io/controller-expand-secret-namespace: hpe-storage
+  csi.storage.k8s.io/controller-publish-secret-name: truenas-secret
+  csi.storage.k8s.io/controller-publish-secret-namespace: hpe-storage
+  csi.storage.k8s.io/node-publish-secret-name: truenas-secret
+  csi.storage.k8s.io/node-publish-secret-namespace: hpe-storage
+  csi.storage.k8s.io/node-stage-secret-name: truenas-secret
+  csi.storage.k8s.io/node-stage-secret-namespace: hpe-storage
+  csi.storage.k8s.io/provisioner-secret-name: truenas-secret
+  csi.storage.k8s.io/provisioner-secret-namespace: hpe-storage
+  csi.storage.k8s.io/fstype: xfs
+  allowOverrides: sparse,compression,deduplication,volblocksize,sync,description  
+  root: zwimming/csi-volumes
+reclaimPolicy: Delete
+allowVolumeExpansion: true
@@ -0,0 +1,38 @@
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: truenas-csp-svc
+  namespace: hpe-storage
+  labels:
+    app: truenas-csp-svc
+spec:
+  ports:
+  - port: 8080
+    protocol: TCP
+  selector:
+    app: truenas-csp
+
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: truenas-csp
+  namespace: hpe-storage
+spec:
+  selector:
+    matchLabels:
+      app: truenas-csp
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: truenas-csp
+    spec:
+      priorityClassName: system-cluster-critical
+      containers:
+        - name: truenas-csp
+          image: quay.io/datamattsson/truenas-csp:v2.3.0
+          imagePullPolicy: IfNotPresent
+          ports:
+          - containerPort: 8080
@@ -0,0 +1,12 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: truenas-secret
+  namespace: hpe-storage
+stringData:
+  serviceName: truenas-csp-svc
+  servicePort: "8080"
+  username: hpe-csi (username is a no-op)
+  password: API key or root password of TrueNAS/FreeNAS appliance
+  backend: Management IP address of TrueNAS/FreeNAS appliance
@@ -29,6 +29,7 @@ Topology is currently not supported by the HPE CSI Driver.
 
 Releases will track the upstream versioning of the HPE CSI Driver for Kubernetes and potential bugfixes in the TrueNAS CSP will be pushed to the same image tag matching the HPE CSI Driver version.
 
+* [TrueNAS CSP v2.3.0](https://github.com/hpe-storage/truenas-csp/releases/tag/v2.3.0) for HPE CSI Driver v2.3.0
 * [TrueNAS CSP v2.2.0](https://github.com/hpe-storage/truenas-csp/releases/tag/v2.2.0) for HPE CSI Driver v2.2.0
 * [TrueNAS CSP v2.1.1](https://github.com/hpe-storage/truenas-csp/releases/tag/v2.1.1) for HPE CSI Driver v2.1.1
 * [TrueNAS CSP v2.1.0](https://github.com/hpe-storage/truenas-csp/releases/tag/v2.1.0) for HPE CSI Driver v2.1.0
@@ -72,14 +73,26 @@ make rwx
 make clean
 ```
 
-**Note:** Not all tests pass at this time.
+**Note:** FreeNAS, TrueNAS CORE and SCALE should pass all tests when configured properly.
+
+## Multiarch build
+
+Since v2.3.0 a multi-arch image is made available, to build locally:
+
+```
+docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 \
+  --provenance=false --push -t truenas-csp:v0.0.0 .
+```
 
 # Limitations
 
 These are the known limitations.
 
 - **Ephemeral Local Volumes:** Due to how TrueNAS handles ZVol names internally and the long names generated by the HPE CSI Driver when requesting ephemeral storage, Ephemeral Local Volumes is not compatible with the TrueNAS CSP. Generic Ephemeral Volumes introduced as an Alpha feature in Kubernetes 1.19 works as the volumes are derived from a regular `StorageClass`.
 - **Volume sizing:** TrueNAS ZVols "volblocksize" need to be even divisible by the requesting volume size. This is a non-issue if you're working with even "Gi" sizes in the `PersistentVolumeClaims`. It gets hairy if working with "Mi" sizes and large volume block sizes.
+- **Dataset naming:** The underscore character `_` is used as an internal separator for naming snapshots and datasets. Do NOT use underscores in your pool or dataset names.
+- **FreeNAS ctl_max_luns:** FreeNAS has an internal limit of 1024 LUNs. That number increments for every new LUN created, even if deleted. The iSCSI Target service won't start and it leads to all sorts of problems. This is the log message on the console: `requested LUN ID 1031 is higher than ctl_max_luns` (this system had two iSCSI Targets).
+- **FreeNAS iSCSI Target:** On systems with a high degree of churn, especially during e2e testing, the iSCSI Target sometimes croak and needs to be restarted. It's recommended to starve the CSP to ease the API requests against FreeNAS and let failures be handled by CSI driver and Kubernetes (see [Helm chart](https://artifacthub.io/packages/helm/truenas-csp/truenas-csp)).
 
 # Need help?
 
@@ -93,7 +106,7 @@ Contributing to the TrueNAS CSP is subject to the following [contributing](CONTR
 
 There's currently no other open source CSPs, but the official HPE CSI Driver for Kubernetes include:
 
-- [HPE Alletra 6000 and Nimble Storage](https://scod.hpedev.io/container_storage_provider/hpe_nimble_storage/index.html)
+- [HPE Alletra 5000/6000 and Nimble Storage](https://scod.hpedev.io/container_storage_provider/hpe_nimble_storage/index.html)
 - [HPE Alletra 9000 and Primera (including 3PAR)](https://scod.hpedev.io/container_storage_provider/hpe_3par_primera/index.html)
 
 # Similar projects
@@ -105,17 +118,17 @@ The TrueNAS CSP is not the only enabler of TrueNAS/FreeNAS for Kubernetes.
 
 # License
 
-TrueNAS(R) (C) 2022 iXsystems, Inc.
+TrueNAS(R) (C) 2023 iXsystems, Inc.
 
-TrueNAS CORE(R) (C) 2022 iXsystems, Inc.
+TrueNAS CORE(R) (C) 2023 iXsystems, Inc.
 
-TrueNAS SCALE(R) (C) 2022 iXsystems, Inc.
+TrueNAS SCALE(R) (C) 2023 iXsystems, Inc.
 
-FreeNAS(R) is (C) 2011-2022 iXsystems
+FreeNAS(R) is (C) 2011-2023 iXsystems
 
 TrueNAS CSP is released under the [MIT License](LICENSE).
 
-(C) Copyright 2022 Hewlett Packard Enterprise Development LP.
+(C) Copyright 2023 Hewlett Packard Enterprise Development LP.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 

@@ -1,6 +1,39 @@
 apiVersion: v1
 entries:
   truenas-csp:
+  - annotations:
+      artifacthub.io/license: MIT
+      artifacthub.io/links: |
+        - name: HPE CSI Driver for Kubernetes
+          url: https://scod.hpedev.io
+        - name: Install
+          url: https://github.com/hpe-storage/truenas-csp/blob/master/INSTALL.md
+      artifacthub.io/prerelease: "false"
+    apiVersion: v2
+    appVersion: 2.3.0
+    created: "2023-05-04T22:52:19.985145376-07:00"
+    dependencies:
+    - name: hpe-csi-driver
+      repository: https://hpe-storage.github.io/co-deployments
+      version: 2.3.0
+    description: TrueNAS Container Storage Provider Helm chart for Kubernetes
+    digest: deef9284f4a876ed74abac395992e33633c33388547ea830dc3788018931a801
+    home: https://github.com/hpe-storage/truenas-csp
+    icon: https://hpe-storage.github.io/truenas-csp/assets/icon.svg
+    keywords:
+    - HPE
+    - Storage
+    - CSI
+    maintainers:
+    - email: michael.mattsson@gmail.com
+      name: Michael Mattsson
+    name: truenas-csp
+    sources:
+    - https://github.com/hpe-storage/truenas-csp
+    type: application
+    urls:
+    - truenas-csp-1.1.3.tgz
+    version: 1.1.3
   - annotations:
       artifacthub.io/license: MIT
       artifacthub.io/links: |
@@ -133,4 +166,4 @@ entries:
     urls:
     - truenas-csp-1.0.0.tgz
     version: 1.0.0
-generated: "2022-09-19T15:42:52.409205899-07:00"
+generated: "2023-05-04T22:52:19.983893272-07:00"
@@ -1,6 +1,7 @@
 E2E=./e2e.test
 TEST_PREFIX=External.Storage.\[Driver:.csi.hpe.com\].*
 FOCUS=
+GINKGO_EXTRA_ARGS=
 SYSTEM=$(shell uname -s | tr A-Z a-z)
 ARCH=$(shell uname -m)
 ifeq ($(ARCH), x86_64)
@@ -25,7 +26,9 @@ endif
 
 rwo: dl
 	mkdir -p reports-rwo
-	$(E2E) --ginkgo.v \
+	$(E2E) $(GINKGO_EXTRA_ARGS) \
+	       --ginkgo.fail-fast \
+	       --ginkgo.v \
 	       --ginkgo.timeout=12h \
                --ginkgo.focus='$(TEST_PREFIX)$(FOCUS)' \
 	       --ginkgo.skip='\[Disruptive\]|\[Serial\]' \
@@ -35,7 +38,9 @@ rwo: dl
 
 rwx: dl
 	mkdir -p reports-rwx
-	$(E2E) --ginkgo.v \
+	$(E2E) $(GINKGO_EXTRA_ARGS) \
+	       --ginkgo.fail-fast \
+	       --ginkgo.v \
 	       --ginkgo.timeout=12h \
                --ginkgo.focus='$(TEST_PREFIX)$(FOCUS)' \
 	       --ginkgo.skip='\[Feature:|\[Disruptive\]|\[Serial\]' \

@@ -2,10 +2,10 @@ StorageClass:
   FromFile: "truenas-csp/e2e/storage-class-rwo.yaml"
 SnapshotClass:
   FromFile: "./volume-snapshot-class.yaml"
-RequiredAccessModes:
-  - ReadWriteOnce
 DriverInfo:
   Name: csi.hpe.com
+  RequiredAccessModes:
+    - ReadWriteOnce
   Capabilities:
     persistence: true
     block: true
@@ -20,6 +20,8 @@ DriverInfo:
     singleNodeVolume: false
     RWX: false
     pvcDataSource: true
+    FSResizeFromSourceNotSupported: true
+    readWriteOncePod: false
   SupportedFsType:
     ext4: {}
     ext3: {}

@@ -1,13 +1,13 @@
 StorageClass:
   FromFile: "truenas-csp/e2e/storage-class-rwx.yaml"
-RequiredAccessModes:
-  - ReadWriteMany
 DriverInfo:
   Name: csi.hpe.com
+  RequiredAccessModes:
+    - ReadWriteMany
   Capabilities:
     persistence: true
     block: false
-    fsGroup: true
+    fsGroup: false
     exec: true
     snapshotDataSource: false
     multipods: true
@@ -18,6 +18,8 @@ DriverInfo:
     singleNodeVolume: false
     RWX: true
     pvcDataSource: false
+    FSResizeFromSourceNotSupported: true
+    readWriteOncePod: false
   SupportedSizeRange: 
     Min: 1Gi 
     Max: 32Gi
@@ -1,6 +1,6 @@
 dependencies:
 - name: hpe-csi-driver
   repository: https://hpe-storage.github.io/co-deployments
-  version: 2.2.0
-digest: sha256:3a0b91d7bf756b55077e024e75f1eb881cff0bc232bc4aaaae28a0d9688eead7
-generated: "2022-09-19T15:42:51.707669049-07:00"
+  version: 2.3.0
+digest: sha256:57a763b1bec68af08c6dca0e9cdced1241eea9aae39481e95843c26784900526
+generated: "2023-05-03T09:32:56.697808632-07:00"
@@ -12,8 +12,8 @@ annotations:
     - name: Install
       url: https://github.com/hpe-storage/truenas-csp/blob/master/INSTALL.md
   artifacthub.io/prerelease: "false"
-version: "1.1.2"
-appVersion: "2.2.0"
+version: "1.1.3"
+appVersion: "2.3.0"
 maintainers:
   - name: Michael Mattsson
     email: michael.mattsson@gmail.com
@@ -22,7 +22,7 @@ sources:
 home: https://github.com/hpe-storage/truenas-csp
 dependencies:
   - name: hpe-csi-driver
-    version: 2.2.0
+    version: 2.3.0
     repository: "https://hpe-storage.github.io/co-deployments"
 keywords:
   - HPE