From 46dbc3e18f7800c35ceadd5142651f0e0efb3f3e Mon Sep 17 00:00:00 2001 From: Hua Huang Date: Wed, 15 Jun 2022 23:40:49 -0600 Subject: [PATCH] Handling 'short write' in some cases, e.g. disk full. In cases, e.g. disk full, the write request may return with 'short write': the written count is less than requested count. The assertion of requested write count equal to returned count is removed. In this case, the rc value of this request is still 0. And next write request may return -ENOSPC because of disk full. This can be triggered with small disks and writing a lot of data into the system. Signed-off-by: Hua Huang --- .xperior/testds/motr-single_tests.yaml | 10 +++ ioservice/io_foms.c | 9 +-- m0t1fs/linux_kernel/st/m0t1fs_common_inc.sh | 2 + m0t1fs/linux_kernel/st/m0t1fs_server_inc.sh | 2 +- motr/st/utils/copy.c | 2 +- motr/st/utils/motr_io_small_disks.sh | 88 +++++++++++++++++++++ motr/st/utils/sns_repair_common_inc.sh | 3 +- scripts/st.d/73motr-io-small-disks | 4 + 8 files changed, 112 insertions(+), 8 deletions(-) create mode 100755 motr/st/utils/motr_io_small_disks.sh create mode 100755 scripts/st.d/73motr-io-small-disks diff --git a/.xperior/testds/motr-single_tests.yaml b/.xperior/testds/motr-single_tests.yaml index f301cd8983e..cae8cb6d070 100644 --- a/.xperior/testds/motr-single_tests.yaml +++ b/.xperior/testds/motr-single_tests.yaml @@ -749,6 +749,16 @@ Tests: polltime : 30 timeout : 1800 + - id : 73motr-io-small-disks + script : 'm0 run-st 73motr-io-small-disks' + dir : src/scripts + executor : Xperior::Executor::MotrTest + sandbox : /var/motr/root/sandbox.st-73motr-io-small-disks + groupname: 04motr-single-node + polltime : 30 + timeout : 1800 + + # 25 min dangerous: 'yes' diff --git a/ioservice/io_foms.c b/ioservice/io_foms.c index 2332fa9307b..e908d358c4b 100644 --- a/ioservice/io_foms.c +++ b/ioservice/io_foms.c @@ -1957,12 +1957,11 @@ static int io_finish(struct m0_fom *fom) } } - M0_LOG(M0_DEBUG, "got fom: %"PRIi64", req_count: %"PRIi64", " - "count: %"PRIx64", nob: %"PRIx64"", fom_obj->fcrw_fom_start_time, - fom_obj->fcrw_req_count, fom_obj->fcrw_count, nob); fom_obj->fcrw_count += nob; - M0_ASSERT(ergo(rc == 0, - fom_obj->fcrw_req_count == fom_obj->fcrw_count)); + M0_LOG(M0_DEBUG, "got fom: %"PRIi64", req_count: %"PRIi64", " + "count: %"PRIx64", nob: %"PRIx64 ", rc:%d", + fom_obj->fcrw_fom_start_time, + fom_obj->fcrw_req_count, fom_obj->fcrw_count, nob, rc); rc = fom_obj->fcrw_rc ?: rc; if (rc != 0) { M0_LOG(M0_ERROR, "rc=%d", rc); diff --git a/m0t1fs/linux_kernel/st/m0t1fs_common_inc.sh b/m0t1fs/linux_kernel/st/m0t1fs_common_inc.sh index fbc3aa131d2..84cc0b5b114 100755 --- a/m0t1fs/linux_kernel/st/m0t1fs_common_inc.sh +++ b/m0t1fs/linux_kernel/st/m0t1fs_common_inc.sh @@ -81,6 +81,8 @@ IOS5_CMD="" #IOS5 process commandline to spawn it again on Controller even IOS4_CMD="" +export IOS_DISK_SEEK_BLOCK_COUNT=1M + # list of md server end points tmid in [800, 899) MDSEP=( 12345:33:800 # MDS1 EP diff --git a/m0t1fs/linux_kernel/st/m0t1fs_server_inc.sh b/m0t1fs/linux_kernel/st/m0t1fs_server_inc.sh index 7f57461ac28..d3825394677 100644 --- a/m0t1fs/linux_kernel/st/m0t1fs_server_inc.sh +++ b/m0t1fs/linux_kernel/st/m0t1fs_server_inc.sh @@ -95,7 +95,7 @@ EOF for (( ; DDEV_ID < $dev_end; DDEV_ID++)) ; do conf_ios_device_setup $DDEV_ID $id_count id_count "$ids" ids - dd if=/dev/zero of=$DDEV_ID$ddisk bs=1M seek=1M count=1 || + dd if=/dev/zero of=$DDEV_ID$ddisk bs=1M seek=$IOS_DISK_SEEK_BLOCK_COUNT count=1 || return 1 if [ ! -e /dev/loop$DDEV_ID ]; then create_loop_device $DDEV_ID diff --git a/motr/st/utils/copy.c b/motr/st/utils/copy.c index 35feace67c2..48dc989f388 100644 --- a/motr/st/utils/copy.c +++ b/motr/st/utils/copy.c @@ -128,7 +128,7 @@ int main(int argc, char **argv) client_fini(m0_instance); - return rc == 0 ? 0 : 1; + return -rc; } /* diff --git a/motr/st/utils/motr_io_small_disks.sh b/motr/st/utils/motr_io_small_disks.sh new file mode 100755 index 00000000000..7a56dc9ea5d --- /dev/null +++ b/motr/st/utils/motr_io_small_disks.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2022 Seagate Technology LLC and/or its Affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# For any questions about this software or licensing, +# please email opensource@seagate.com or cortx-questions@seagate.com. +# + + +TOPDIR=$(dirname "$0")/../../../ + +. "${TOPDIR}/m0t1fs/linux_kernel/st/common.sh" +. "${TOPDIR}/m0t1fs/linux_kernel/st/m0t1fs_common_inc.sh" +. "${TOPDIR}/m0t1fs/linux_kernel/st/m0t1fs_client_inc.sh" +. "${TOPDIR}/m0t1fs/linux_kernel/st/m0t1fs_server_inc.sh" +. "${TOPDIR}/m0t1fs/linux_kernel/st/m0t1fs_sns_common_inc.sh" +. "${TOPDIR}/motr/st/utils/sns_repair_common_inc.sh" + + +export MOTR_CLIENT_ONLY=1 + +# The ioservice will have a very small disk to trigger -ENOSPC test +export IOS_DISK_SEEK_BLOCK_COUNT=500 + +motr_io_small_disks() +{ + local rc=0 + + echo "Startin motr_io_small_disks testing ..." + + prepare_datafiles_and_objects + rc=$? + if [[ $rc -eq 28 ]] ; then + # ENOSPC == 28 + echo "We got $rc as expected" + rc=0 + else + echo "We got $rc. This is not expected." + rc=1 + fi + + return $? +} + +main() +{ + local rc=0 + + sandbox_init + + NODE_UUID=$(uuidgen) + local multiple_pools=0 + motr_service start $multiple_pools $stride $N $K $S $P || { + echo "Failed to start Motr Service." + return 1 + } + + + if [[ $rc -eq 0 ]] && ! motr_io_small_disks ; then + rc=1 + echo "Failed: Motr I/O testing on small disks.." + fi + + motr_service stop || { + echo "Failed to stop Motr Service." + rc=1 + } + if [[ $rc -eq 0 ]]; then + sandbox_fini + fi + return $rc +} + +trap unprepare EXIT +main +report_and_exit motr_io_small_disks $? diff --git a/motr/st/utils/sns_repair_common_inc.sh b/motr/st/utils/sns_repair_common_inc.sh index 51598b30d34..d1cffab559c 100644 --- a/motr/st/utils/sns_repair_common_inc.sh +++ b/motr/st/utils/sns_repair_common_inc.sh @@ -42,7 +42,8 @@ prepare_datafiles_and_objects() -o ${file[$i]} \ "$MOTR_M0T1FS_TEST_DIR/srcfile" || { rc=$? - echo "Writing object ${file[$i]} failed" + echo "Writing object ${file[$i]} failed: $rc" + return $rc } done return $rc diff --git a/scripts/st.d/73motr-io-small-disks b/scripts/st.d/73motr-io-small-disks new file mode 100755 index 00000000000..a64a7c97474 --- /dev/null +++ b/scripts/st.d/73motr-io-small-disks @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -eu + +exec $SUDO "$M0_SRC_DIR/motr/st/utils/motr_io_small_disks.sh"