Merge branch 'main' into csi_nil_pointer_check_new

Signed-off-by: Xun Jiang/Bruce Jiang <59276555+blackpiglet@users.noreply.github.com>

.github/workflows/crds-verify-kind.yaml

@@ -14,7 +14,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.17
+          go-version: 1.18
         id: go
       # Look for a CLI that's made for this PR
       - name: Fetch built CLI

.github/workflows/e2e-test-kind.yaml

@@ -14,7 +14,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.17
+          go-version: 1.18
         id: go
       # Look for a CLI that's made for this PR
       - name: Fetch built CLI
@@ -76,7 +76,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.17
+          go-version: 1.18
         id: go
       - name: Check out the code
         uses: actions/checkout@v2
@@ -117,12 +117,17 @@ jobs:
           aws_access_key_id=minio
           aws_secret_access_key=minio123
           EOF
+
+          # Match kubectl version to k8s server version
+          curl -LO https://dl.k8s.io/release/v${{ matrix.k8s }}/bin/linux/amd64/kubectl
+          sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+
           GOPATH=~/go CLOUD_PROVIDER=kind \
               OBJECT_STORE_PROVIDER=aws BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \
               CREDS_FILE=/tmp/credential BSL_BUCKET=bucket \
               ADDITIONAL_OBJECT_STORE_PROVIDER=aws ADDITIONAL_BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \
               ADDITIONAL_CREDS_FILE=/tmp/credential ADDITIONAL_BSL_BUCKET=additional-bucket \
-              GINKGO_FOCUS='Basic\].+\[ClusterResource' VELERO_IMAGE=velero:pr-test \
+              GINKGO_FOCUS='Basic\]\[ClusterResource' VELERO_IMAGE=velero:pr-test \
               make -C test/e2e run
         timeout-minutes: 30
       - name: Upload debug bundle

.github/workflows/pr-ci-check.yml

@@ -8,7 +8,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.17
+          go-version: 1.18
         id: go
       - name: Check out the code
         uses: actions/checkout@v2

.github/workflows/pr-codespell.yml

@@ -15,6 +15,6 @@ jobs:
       with:
         # ignore the config/.../crd.go file as it's generated binary data that is edited elswhere.
         skip: .git,*.png,*.jpg,*.woff,*.ttf,*.gif,*.ico,./config/crd/v1beta1/crds/crds.go,./config/crd/v1/crds/crds.go,./go.sum,./LICENSE
-        ignore_words_list: iam,aks,ist,bridget,ue
+        ignore_words_list: iam,aks,ist,bridget,ue,shouldnot
         check_filenames: true
         check_hidden: true

.github/workflows/push.yml

@@ -18,7 +18,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.17
+        go-version: 1.18
       id: go
 
     - name: Check out code into the Go module directory

.gitignore

@@ -46,7 +46,7 @@ _tiltbuild
 tilt-resources/tilt-settings.json
 tilt-resources/velero_v1_backupstoragelocation.yaml
 tilt-resources/deployment.yaml
-tilt-resources/restic.yaml
+tilt-resources/node-agent.yaml
 tilt-resources/cloud
 
 test/e2e/report.xml

Dockerfile

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-FROM --platform=$BUILDPLATFORM golang:1.17 as builder-env
+FROM --platform=$BUILDPLATFORM golang:1.18 as builder-env
 
 ARG GOPROXY
 ARG PKG

MAINTAINERS.md

@@ -30,9 +30,10 @@
 
 | Feature Area | Lead |
 | ----------------------------- | :---------------------: |
-| Architect | Dave Smith-Uchida (dsu-igeek) |
-| Technical Lead | Daniel Jiang (reasonerjt) |
+| Architect | Dave Smith-Uchida [dsu-igeek](https://github.com/dsu-igeek) |
+| Technical Lead | Daniel Jiang [reasonerjt](https://github.com/reasonerjt) |
 | Kubernetes CSI Liaison |  |
 | Deployment |  |
-| Community Management | Orlin Vasilev (OrlinVasilev) |
-| Product Management | Eleanor Millman (eleanor-millman) |
+| Community Management | Orlin Vasilev [OrlinVasilev](https://github.com/OrlinVasilev) |
+| Product Management | Pradeep Kumar Chaturvedi [pradeepkchaturvedi](https://github.com/pradeepkchaturvedi) |
+

Makefile

@@ -112,17 +112,17 @@ GOPROXY ?= https://proxy.golang.org
 # If you want to build all containers, see the 'all-containers' rule.
 all:
 	@$(MAKE) build
-	@$(MAKE) build BIN=velero-restic-restore-helper
+	@$(MAKE) build BIN=velero-restore-helper
 
 build-%:
 	@$(MAKE) --no-print-directory ARCH=$* build
-	@$(MAKE) --no-print-directory ARCH=$* build BIN=velero-restic-restore-helper
+	@$(MAKE) --no-print-directory ARCH=$* build BIN=velero-restore-helper
 
 all-build: $(addprefix build-, $(CLI_PLATFORMS))
 
 all-containers: container-builder-env
 	@$(MAKE) --no-print-directory container
-	@$(MAKE) --no-print-directory container BIN=velero-restic-restore-helper
+	@$(MAKE) --no-print-directory container BIN=velero-restore-helper
 
 local: build-dirs
 # Add DEBUG=1 to enable debug locally

Tiltfile

@@ -17,7 +17,7 @@ k8s_yaml([
 # default values
 settings = {
     "default_registry": "docker.io/velero",
-    "enable_restic": False,
+    "use_node_agent": False,
     "enable_debug": False,
     "debug_continue_on_start": True,  # Continue the velero process by default when in debug mode
     "create_backup_locations": False,
@@ -34,9 +34,9 @@ k8s_yaml(kustomize('tilt-resources'))
 k8s_yaml('tilt-resources/deployment.yaml')
 if settings.get("enable_debug"):
     k8s_resource('velero', port_forwards = '2345')
-    # TODO: Need to figure out how to apply port forwards for all restic pods
-if settings.get("enable_restic"):
-    k8s_yaml('tilt-resources/restic.yaml')
+    # TODO: Need to figure out how to apply port forwards for all node-agent pods
+if settings.get("use_node_agent"):
+    k8s_yaml('tilt-resources/node-agent.yaml')
 if settings.get("create_backup_locations"):
     k8s_yaml('tilt-resources/velero_v1_backupstoragelocation.yaml')
 if settings.get("setup-minio"):
@@ -50,7 +50,7 @@ git_sha = str(local("git rev-parse HEAD", quiet = True, echo_off = True)).strip(
 
 tilt_helper_dockerfile_header = """
 # Tilt image
-FROM golang:1.17 as tilt-helper
+FROM golang:1.18 as tilt-helper
 
 # Support live reloading with Tilt
 RUN wget --output-document /restart.sh --quiet https://raw.githubusercontent.com/windmilleng/rerun-process-wrapper/master/restart.sh  && \

changelogs/unreleased/5064-jxun

@@ -0,0 +1 @@
+Exclude "csinodes.storage.k8s.io" and "volumeattachments.storage.k8s.io" from restore by default.

changelogs/unreleased/5283-blackpiglet

@@ -0,0 +1 @@
+Add backup status checking in schedule controller.

changelogs/unreleased/5331-danfengliu

@@ -0,0 +1 @@
+Add opt-in and opt-out PersistentVolume backup to E2E tests

changelogs/unreleased/5362-niulechuan

@@ -0,0 +1 @@
+Added backupController's UT to test the prepareBackupRequest() method BackupStorageLocation processing logic

changelogs/unreleased/5377-qiuming-best

@@ -0,0 +1 @@
+Fix restore error with flag namespace-mappings

changelogs/unreleased/5387-lyndon

@@ -0,0 +1 @@
+Fix issue 5386: Velero providers a full URL as the S3Url while the underlying minio client only accept the host part of the URL as the endpoint and the schema should be specified separately.

changelogs/unreleased/5390-lyndon

@@ -0,0 +1 @@
+Rename Velero daemonset from "restic" to "node-agent"

changelogs/unreleased/5394-blackpiglet

@@ -0,0 +1 @@
+Fix Test_prepareBackupRequest_BackupStorageLocation UT failure.

changelogs/unreleased/5396-blackpiglet

@@ -0,0 +1 @@
+update velero using klog to version v2.9.0

changelogs/unreleased/5412-allenxu404

@@ -0,0 +1 @@
+Change B/R describe CLI to support Kopia

changelogs/unreleased/5429-reasonerjt

@@ -0,0 +1 @@
+Skip the exclusion check for additional resources returned by BIA

changelogs/unreleased/5432-lyndon

@@ -0,0 +1 @@
+Rename Velero pod volume restore init helper from "velero-restic-restore-helper" to "velero-restore-helper"

changelogs/unreleased/5441-sseago

@@ -0,0 +1 @@
+moved RIA execute input/output structs back to velero package

changelogs/unreleased/5444-lyndon

@@ -0,0 +1 @@
+Remove irrational "Restic" names in Velero code after the PVBR refactor

changelogs/unreleased/5446-allenxu404

@@ -0,0 +1 @@
+Change subcommand `velero restic repo` to `velero repo`

changelogs/unreleased/5449-blackpiglet

@@ -0,0 +1 @@
+Add VolumeSnapshot client back.

cmd/velero/velero.go

@@ -20,7 +20,7 @@ import (
 	"os"
 	"path/filepath"
 
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 
 	"github.com/vmware-tanzu/velero/pkg/cmd"
 	"github.com/vmware-tanzu/velero/pkg/cmd/velero"

config/crd/v1/bases/velero.io_podvolumerestores.yaml

@@ -116,6 +116,10 @@ spec:
               snapshotID:
                 description: SnapshotID is the ID of the volume snapshot to be restored.
                 type: string
+              sourceNamespace:
+                description: SourceNamespace is the original namespace for namaspace
+                  mapping.
+                type: string
               uploaderType:
                 description: UploaderType is the type of the uploader to handle the
                   data transfer.
@@ -133,6 +137,7 @@ spec:
             - pod
             - repoIdentifier
             - snapshotID
+            - sourceNamespace
             - volume
             type: object
           status:

design/CLI/PoC/overlays/plugins/restic.yaml → design/CLI/PoC/overlays/plugins/node-agent.yaml

@@ -5,22 +5,22 @@ metadata:
   creationTimestamp: null
   labels:
     component: velero
-  name: restic
+  name: node-agent
   namespace: velero
 spec:
   selector:
     matchLabels:
-      name: restic
+      name: node-agent
   template:
     metadata:
       creationTimestamp: null
       labels:
         component: velero
-        name: restic
+        name: node-agent
     spec:
       containers:
         - args:
-            - restic
+            - node-agent
             - server
           command:
             - /velero
@@ -43,7 +43,7 @@ spec:
               value: /credentials/cloud
           image: velero/velero:latest
           imagePullPolicy: Always
-          name: restic
+          name: node-agent
           resources: {}
           volumeMounts:
             - mountPath: /host_pods

design/Implemented/backup-resources-order.md

@@ -2,7 +2,7 @@
 This document proposes a solution that allows user to specify a backup order for resources of specific resource type.
 
 ## Background
-During backup process, user may need to back up resources of specific type in some specific order to ensure the resources were backup properly because these resources are related and ordering might be required to preserve the consistency for the apps to recover itself  from the backup image 
+During backup process, user may need to back up resources of specific type in some specific order to ensure the resources were backup properly because these resources are related and ordering might be required to preserve the consistency for the apps to recover itself �from the backup image 
 (Ex: primary-secondary database pods in a cluster).
 
 ## Goals
@@ -12,7 +12,7 @@ During backup process, user may need to back up resources of specific type in so
 - Use a plugin to backup an resources and all the sub resources.  For example use a plugin for StatefulSet and backup pods belong to the StatefulSet in specific order.  This plugin solution is not generic and requires plugin for each resource type.
 
 ## High-Level Design
-User will specify a map of resource type to list resource names (separate by semicolons).  Each name will be in the format "namespaceName/resourceName" to enable ordering accross namespaces.  Based on this map, the resources of each resource type will be sorted by the order specified in the list of resources.  If a resource instance belong to that specific type but its name is not in the order list, then it will be put behind other resources that are in the list.
+User will specify a map of resource type to list resource names (separate by semicolons).  Each name will be in the format "namespaceName/resourceName" to enable ordering across namespaces.  Based on this map, the resources of each resource type will be sorted by the order specified in the list of resources.  If a resource instance belong to that specific type but its name is not in the order list, then it will be put behind other resources that are in the list.
 
 ### Changes to BackupSpec
 Add new field to BackupSpec
@@ -36,5 +36,5 @@ Example:
 >velero backup create mybackup --ordered-resources "pod=ns1/pod1,ns1/pod2;persistentvolumeclaim=n2/slavepod,ns2/primarypod"
 
 ## Open Issues
-- In the CLI, the design proposes to use commas to separate items of a resource type and semicolon to separate key-value pairs.  This follows the convention of using commas to separate items in a list (For example: --include-namespaces ns1,ns2).  However, the syntax for map in labels and annotations use commas to seperate key-value pairs.  So it introduces some inconsistency.
+- In the CLI, the design proposes to use commas to separate items of a resource type and semicolon to separate key-value pairs.  This follows the convention of using commas to separate items in a list (For example: --include-namespaces ns1,ns2).  However, the syntax for map in labels and annotations use commas to separate key-value pairs.  So it introduces some inconsistency.
 - For pods that managed by Deployment or DaemonSet, this design may not work because the pods' name is randomly generated and if pods are restarted, they would have different names so the Backup operation may not consider the restarted pods in the sorting algorithm.  This problem will be addressed when we enhance the design to use regular expression to specify the OrderResources instead of exact match. 

design/Implemented/generating-velero-crds-with-structural-schema.md

@@ -28,7 +28,7 @@ This document proposes adding _controller-tools_ to the project to automatically
 _controller-tools_ works by reading the Go files that contain the API type definitions.
 It uses a combination of the struct fields, types, tags and comments to build the OpenAPIv3 schema for the CRDs. The tooling makes some assumptions based on conventions followed in upstream Kubernetes and the ecosystem, which involves some changes to the Velero API type definitions, especially around optional fields.
 
-In order for _controller-tools_ to read the Go files containing Velero API type defintiions, the CRDs need to be generated at build time, as these files are not available at runtime (i.e. the Go files are not accessible by the compiled binary).
+In order for _controller-tools_ to read the Go files containing Velero API type definitions, the CRDs need to be generated at build time, as these files are not available at runtime (i.e. the Go files are not accessible by the compiled binary).
 These generated CRD manifests (YAML) will then need to be available to the `pkg/install` package for it to include when installing Velero resources.
 
 ## Detailed Design

go.mod

@@ -1,6 +1,6 @@
 module github.com/vmware-tanzu/velero
 
-go 1.17
+go 1.18
 
 require (
 	cloud.google.com/go/storage v1.21.0
@@ -47,7 +47,7 @@ require (
 	k8s.io/apimachinery v0.22.2
 	k8s.io/cli-runtime v0.22.2
 	k8s.io/client-go v0.22.2
-	k8s.io/klog v1.0.0
+	k8s.io/klog/v2 v2.9.0
 	k8s.io/kube-aggregator v0.19.12
 	sigs.k8s.io/controller-runtime v0.10.2
 	sigs.k8s.io/yaml v1.3.0
@@ -139,7 +139,6 @@ require (
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 	k8s.io/component-base v0.22.2 // indirect
-	k8s.io/klog/v2 v2.9.0 // indirect
 	k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e // indirect
 	k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect