CORTX-32116: Handling 'short write' in some cases, e.g. disk full. (S…

…eagate#1888)

In cases, e.g. disk full, the write request may return
with 'short write': the written count is less than requested count.
The assertion of requested write count equal to returned count is removed.
In this case, the rc value of this request is still 0. And next write
request may return -ENOSPC because of disk full.
This can be triggered with small disks and writing a lot of data into
the system.

Signed-off-by: Hua Huang <hua.huang@seagate.com>

.xperior/testds/motr-single_tests.yaml

@@ -749,6 +749,16 @@ Tests:
     polltime : 30
     timeout  : 1800
 
+  - id       : 73motr-io-small-disks
+    script   : 'm0 run-st 73motr-io-small-disks'
+    dir      : src/scripts
+    executor : Xperior::Executor::MotrTest
+    sandbox  : /var/motr/root/sandbox.st-73motr-io-small-disks
+    groupname: 04motr-single-node
+    polltime : 30
+    timeout  : 1800
+
+
 
 # 25 min
 dangerous: 'yes'

ioservice/io_foms.c

@@ -1957,12 +1957,11 @@ static int io_finish(struct m0_fom *fom)
 		}
 	}
 
-	M0_LOG(M0_DEBUG, "got    fom: %"PRIi64", req_count: %"PRIi64", "
-	       "count: %"PRIx64", nob: %"PRIx64"", fom_obj->fcrw_fom_start_time,
-	       fom_obj->fcrw_req_count, fom_obj->fcrw_count, nob);
 	fom_obj->fcrw_count += nob;
-	M0_ASSERT(ergo(rc == 0,
-		       fom_obj->fcrw_req_count == fom_obj->fcrw_count));
+	M0_LOG(M0_DEBUG, "got    fom: %"PRIi64", req_count: %"PRIi64", "
+			 "count: %"PRIx64", nob: %"PRIx64 ", rc:%d",
+			 fom_obj->fcrw_fom_start_time,
+			 fom_obj->fcrw_req_count, fom_obj->fcrw_count, nob, rc);
 	rc = fom_obj->fcrw_rc ?: rc;
 	if (rc != 0) {
 		M0_LOG(M0_ERROR, "rc=%d", rc);

m0t1fs/linux_kernel/st/m0t1fs_common_inc.sh

@@ -81,6 +81,8 @@ IOS5_CMD=""       #IOS5 process commandline to spawn it again on Controller even
 
 IOS4_CMD=""
 
+export IOS_DISK_SEEK_BLOCK_COUNT=1M
+
 # list of md server end points tmid in [800, 899)
 MDSEP=(
     12345:33:800   # MDS1 EP

m0t1fs/linux_kernel/st/m0t1fs_server_inc.sh

@@ -95,7 +95,7 @@ EOF
 	for (( ; DDEV_ID < $dev_end; DDEV_ID++)) ; do
 		conf_ios_device_setup $DDEV_ID $id_count id_count "$ids" ids
 
-		dd if=/dev/zero of=$DDEV_ID$ddisk bs=1M seek=1M count=1 ||
+		dd if=/dev/zero of=$DDEV_ID$ddisk bs=1M seek=$IOS_DISK_SEEK_BLOCK_COUNT count=1 ||
 			return 1
 		if [ ! -e /dev/loop$DDEV_ID ]; then
 			create_loop_device $DDEV_ID

motr/st/utils/copy.c

@@ -128,7 +128,7 @@ int main(int argc, char **argv)
 
 	client_fini(m0_instance);
 
-	return rc == 0 ? 0 : 1;
+	return -rc;
 }
 
 /*

motr/st/utils/motr_io_small_disks.sh

@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2022 Seagate Technology LLC and/or its Affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# For any questions about this software or licensing,
+# please email opensource@seagate.com or cortx-questions@seagate.com.
+#
+
+
+TOPDIR=$(dirname "$0")/../../../
+
+. "${TOPDIR}/m0t1fs/linux_kernel/st/common.sh"
+. "${TOPDIR}/m0t1fs/linux_kernel/st/m0t1fs_common_inc.sh"
+. "${TOPDIR}/m0t1fs/linux_kernel/st/m0t1fs_client_inc.sh"
+. "${TOPDIR}/m0t1fs/linux_kernel/st/m0t1fs_server_inc.sh"
+. "${TOPDIR}/m0t1fs/linux_kernel/st/m0t1fs_sns_common_inc.sh"
+. "${TOPDIR}/motr/st/utils/sns_repair_common_inc.sh"
+
+
+export MOTR_CLIENT_ONLY=1
+
+# The ioservice will have a very small disk to trigger -ENOSPC test
+export IOS_DISK_SEEK_BLOCK_COUNT=500
+
+motr_io_small_disks()
+{
+	local rc=0
+
+	echo "Startin motr_io_small_disks testing ..."
+
+	prepare_datafiles_and_objects
+	rc=$?
+	if [[ $rc -eq 28 ]] ; then
+		# ENOSPC == 28
+		echo "We got $rc as expected"
+		rc=0
+	else
+		echo "We got $rc. This is not expected."
+		rc=1
+	fi
+
+	return $?
+}
+
+main()
+{
+	local rc=0
+
+	sandbox_init
+
+	NODE_UUID=$(uuidgen)
+	local multiple_pools=0
+	motr_service start $multiple_pools $stride $N $K $S $P || {
+		echo "Failed to start Motr Service."
+		return 1
+	}
+
+
+	if [[ $rc -eq 0 ]] && ! motr_io_small_disks ; then
+		rc=1
+		echo "Failed: Motr I/O testing on small disks.."
+	fi
+
+	motr_service stop || {
+		echo "Failed to stop Motr Service."
+		rc=1
+	}
+	if [[ $rc -eq 0 ]]; then
+		sandbox_fini
+	fi
+	return $rc
+}
+
+trap unprepare EXIT
+main
+report_and_exit motr_io_small_disks $?

motr/st/utils/sns_repair_common_inc.sh

@@ -42,7 +42,8 @@ prepare_datafiles_and_objects()
 						 -o ${file[$i]}      \
 					 "$MOTR_M0T1FS_TEST_DIR/srcfile" || {
 			rc=$?
-			echo "Writing object ${file[$i]} failed"
+			echo "Writing object ${file[$i]} failed: $rc"
+			return $rc
 		}
 	done
 	return $rc

scripts/st.d/73motr-io-small-disks

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -eu
+
+exec $SUDO "$M0_SRC_DIR/motr/st/utils/motr_io_small_disks.sh"