QA-35128 zpool import azure blob support (openzfs#513)

* Support for running tests with azure blob

PR URL: https://www.github.com/delphix/zfs/pull/513
- zpool import object store tests.
- background freeing.
- zpool destroy object store tests.

tests/zfs-tests/include/commands.cfg

@@ -11,6 +11,7 @@
 export SYSTEM_FILES_COMMON='arp
     awk
     aws
+    az
     base64
     basename
     bc

tests/zfs-tests/include/object_store.shlib

@@ -71,6 +71,61 @@ function pool_exists_s3
 	return 1
 }
 
+#
+# Returns if a pool exists in the azure blob.
+#
+function pool_exists_blob
+{
+	typeset guid=$1
+
+	# Max num of retries (default 5)
+	typeset num_retries=${2:-5}
+
+	# Constant backoff duration (default 2s)
+	typeset retry_backoff_duration=${3:-2}
+
+	typeset retry_count=0
+
+	# Check if the pool exists in the azure blob.
+	# If the exit code is 0, return immediately
+	# else retry for the given duration (default 10s)
+	while [ $retry_count -le $num_retries ]; do
+		count=$(az storage blob list \
+			--container-name $ZTS_BUCKET_NAME \
+			--account-name $AZURE_ACCOUNT \
+			--account-key $AZURE_KEY \
+			--prefix  zfs/$guid/ \
+			--output table \
+			| awk -F '/' '/zfs/ {print $2}' \
+			| sort -u | wc -l)
+
+		[ $count -eq 1 ] && return 0
+		retry_count=$((retry_count + 1))
+		sleep $retry_backoff_duration
+	done
+	return 1
+}
+
+#
+# Returns that the pool corresponding to the guid
+# exists in the object store
+#
+function pool_exists_object_store
+{
+	typeset guid=$1
+	typeset -i rc=1
+
+	if [ $ZTS_OBJECT_STORE == "s3" ]; then
+		pool_exists_s3 $guid
+		rc=$?
+	elif [ $ZTS_OBJECT_STORE == "blob" ]; then
+		pool_exists_blob $guid
+		rc=$?
+	fi
+	return $rc
+}
+
+
 #
 # Verify if the object store pool is online. Verify the allocated space.
 # Also verify that the pool exists in the object store bucket.
@@ -98,8 +153,8 @@ function verify_active_object_store_pool # pool guid exp_allocated
 	done
 	log_must [ $is_allocated -eq 1 ]
 
-	# Verify that the pool exists in the bucket
-	log_must pool_exists_s3 $guid
+	# Verify that the pool exists in the object store
+	log_must pool_exists_object_store $guid
 }
 
 #
@@ -170,8 +225,8 @@ function verify_destroyed_object_store_pool # pool guid
 	# Verify that the pool GUID should be part of zpool_destroy.cache
 	log_must cat /etc/zfs/zpool_destroy.cache | grep $guid
 
-	# Verify that the pool no longer exists in the bucket
-	log_mustnot pool_exists_s3 $guid
+	# Verify that the pool no longer exists in the object store
+	log_mustnot pool_exists_object_store $guid
 }
 
 #

tests/zfs-tests/tests/functional/background_freeing/default_free_space_reclaimation.ksh

@@ -91,8 +91,23 @@ typeset guid=$(get_object_store_pool_guid $TESTPOOL)
 #
 log_must test "$guid" != "00000000000000000000"
 
-typeset num_pending_frees_splits=$(aws --endpoint-url $ZTS_OBJECT_ENDPOINT \
-	s3 ls $ZTS_BUCKET_NAME/zfs/$guid/PendingFreesLog/ | wc -l)
+typeset -i num_pending_frees_splits=0
+
+if [ $ZTS_OBJECT_STORE == "s3" ]; then
+	num_pending_frees_splits=$(aws --endpoint-url $ZTS_OBJECT_ENDPOINT \
+		s3 ls $ZTS_BUCKET_NAME/zfs/$guid/PendingFreesLog/ | wc -l)
+elif [ $ZTS_OBJECT_STORE == "blob" ]; then
+	# This list all objects, to get the split count
+	# extract the 4th column separated by '/'
+	# zfs/08878726368137311436/PendingFreesLog/00000/00000000000000000000/...
+	num_pending_frees_splits=$(az storage blob list -c $ZTS_BUCKET_NAME \
+		--account-name $AZURE_ACCOUNT --account-key $AZURE_KEY \
+		--output table \
+		--prefix zfs/$guid/PendingFrees 2>/dev/null \
+		| awk -F '/' '/zfs/ {print $4}' \
+		| sort -u | wc -l)
+fi
+
 log_note "Total no of pending frees log split $num_pending_frees_splits"
 #
 # The pending frees log can hold approximately 10 million objects
@@ -109,8 +124,18 @@ log_must test $num_pending_frees_splits -gt 0
 # A recursive call to list the parent object (PendingFreesLog)
 # can help in summarizing the child count
 #
-typeset num_pending_frees_objects=$(aws --endpoint-url $ZTS_OBJECT_ENDPOINT \
-	s3 ls $ZTS_BUCKET_NAME/zfs/$guid/PendingFreesLog/ --recursive | wc -l)
+typeset -i num_pending_frees_objects=0
+
+if [ $ZTS_OBJECT_STORE == "s3" ]; then
+	num_pending_frees_objects=$(aws --endpoint-url $ZTS_OBJECT_ENDPOINT \
+		s3 ls $ZTS_BUCKET_NAME/zfs/$guid/PendingFreesLog/ --recursive | wc -l)
+elif [ $ZTS_OBJECT_STORE == "blob" ]; then
+	num_pending_frees_objects=$(az storage blob list -c $ZTS_BUCKET_NAME \
+		--account-name $AZURE_ACCOUNT --account-key $AZURE_KEY \
+		--output table \
+		--prefix zfs/$guid/PendingFrees 2>/dev/null | awk '/zfs/' | wc -l)
+fi
+
 log_note "Total no of pending frees objects $num_pending_frees_objects"
 log_must test $num_pending_frees_objects -gt 0
 

tests/zfs-tests/tests/functional/cli_root/zpool_import/import_agent_stopped_multi_pools_object_store.ksh

@@ -78,14 +78,23 @@ typeset pid=$!
 
 log_note "Started the zfs-object-agent in background with pid=$pid"
 
-typeset OBJECT_STORE_PARAMS="-o object-endpoint=$ZTS_OBJECT_ENDPOINT \
-    -o object-region=$ZTS_REGION \
-    -o object-credentials-profile=${ZTS_CREDS_PROFILE:-default}"
+
+typeset object_store_params="-o object-endpoint=$ZTS_OBJECT_ENDPOINT
+	-o object-region=$ZTS_REGION"
+
+if [ $ZTS_OBJECT_STORE == "s3" ]; then
+	if ! is_using_iam_role; then
+		object_store_params="$object_store_params
+			-o object-credentials-profile=${ZTS_CREDS_PROFILE:-default}"
+	fi
+elif [ $ZTS_OBJECT_STORE == "blob" ]; then
+	object_store_params="-o object-protocol=blob"
+fi
 
 # The pool import should hang at this moment since there
 # exists the default TESTPOOL. This import should resume once the object
 # agent starts after a minute.
-log_must zpool import $OBJECT_STORE_PARAMS \
+log_must zpool import $object_store_params \
 	-d $ZTS_BUCKET_NAME $TESTPOOL1
 
 

tests/zfs-tests/tests/functional/cli_root/zpool_import/import_agent_stopped_object_store.ksh

@@ -95,17 +95,25 @@ typeset pid=$!
 log_note "Background process to start object agent in 2 mins started" \
 	" with pid: $pid"
 
-typeset OBJECT_STORE_PARAMS="-o object-endpoint=$ZTS_OBJECT_ENDPOINT \
-    -o object-region=$ZTS_REGION \
-    -o object-credentials-profile=${ZTS_CREDS_PROFILE:-default}"
+typeset object_store_params="-o object-endpoint=$ZTS_OBJECT_ENDPOINT
+	-o object-region=$ZTS_REGION"
+
+if [ $ZTS_OBJECT_STORE == "s3" ]; then
+	if ! is_using_iam_role; then
+		object_store_params="$object_store_params
+			-o object-credentials-profile=${ZTS_CREDS_PROFILE:-default}"
+	fi
+elif [ $ZTS_OBJECT_STORE == "blob" ]; then
+	object_store_params="-o object-protocol=blob"
+fi
 
 # This should retry for 15 times before giving up
 
 # Use the zpool import command directly instead of the
 # library function import_pool since it makes use of
 # zpool get command to check if pool exists and the
 # command hangs if the object agent is not running
-log_mustnot zpool import $OBJECT_STORE_PARAMS \
+log_mustnot zpool import $object_store_params \
 	-d $ZTS_BUCKET_NAME $TESTPOOL
 
 #