From 1cbeb5331507b9a0c2e993946599374b8ffa8b3c Mon Sep 17 00:00:00 2001 From: Dalton Bohning Date: Mon, 9 Oct 2023 19:34:04 +0000 Subject: [PATCH 1/8] DAOS-12859 test: use pool and container labels (pass 3) Test-tag: ContainerQueryAttributeTest ManagementServiceResilience test_bashcmd ParallelIo NvmePoolCapacity DestroyTests ListPoolsTest RbldBasic RbldBasic RbldPoolDestroyWithIO RbldReadArrayTest RbldWithIO ObjectMetadata performance,-manual container,pr DaosVolBigIO DaosVol MacsioTest TestWithScrubberBasic TestWithScrubberFault TestWithScrubberTargetEviction dfuse datamover test_container_redundancy_factor_oclass_enforcement RbldContRfTest osa RbldCascadingFailures RbldDeleteObjects RbldReadArrayTest Skip-unit-tests: true Skip-fault-injection-test: true Required-githooks: true Signed-off-by: Dalton Bohning --- src/tests/ftest/container/rf_enforcement.py | 10 +---- src/tests/ftest/control/ms_resilience.py | 4 +- src/tests/ftest/dfuse/bash.py | 2 +- src/tests/ftest/io/macsio_test.py | 28 ++++++------ src/tests/ftest/io/parallel_io.py | 2 +- src/tests/ftest/nvme/pool_capacity.py | 3 +- src/tests/ftest/pool/destroy.py | 5 +-- src/tests/ftest/pool/list_pools.py | 27 ++++++------ src/tests/ftest/pool/query_attribute.py | 17 +++++--- src/tests/ftest/rebuild/basic.py | 4 +- src/tests/ftest/rebuild/cascading_failures.py | 6 +-- src/tests/ftest/rebuild/container_rf.py | 3 +- src/tests/ftest/rebuild/no_cap.py | 3 +- src/tests/ftest/rebuild/pool_destroy_race.py | 6 +-- src/tests/ftest/rebuild/read_array.py | 10 +---- src/tests/ftest/rebuild/with_io.py | 9 ++-- src/tests/ftest/scrubber/basic.py | 6 +-- src/tests/ftest/scrubber/csum_fault.py | 2 +- .../ftest/scrubber/target_auto_eviction.py | 6 +-- src/tests/ftest/server/metadata.py | 10 ++--- src/tests/ftest/util/apricot/apricot/test.py | 7 ++- .../ftest/util/container_rf_test_base.py | 18 +++----- src/tests/ftest/util/data_mover_test_base.py | 8 ++-- src/tests/ftest/util/dfuse_utils.py | 2 +- src/tests/ftest/util/macsio_test_base.py | 12 +++--- src/tests/ftest/util/osa_utils.py | 4 +- src/tests/ftest/util/performance_test_base.py | 4 +- src/tests/ftest/util/rebuild_test_base.py | 9 +--- src/tests/ftest/util/scrubber_test_base.py | 43 +++++++------------ src/tests/ftest/util/test_utils_container.py | 4 +- src/tests/ftest/util/vol_test_base.py | 10 ++--- 31 files changed, 119 insertions(+), 165 deletions(-) diff --git a/src/tests/ftest/container/rf_enforcement.py b/src/tests/ftest/container/rf_enforcement.py index dcacf091398..713049a5d1e 100644 --- a/src/tests/ftest/container/rf_enforcement.py +++ b/src/tests/ftest/container/rf_enforcement.py @@ -13,11 +13,6 @@ class ContRfEnforce(ContRedundancyFactor): :avocado: recursive """ - def __init__(self, *args, **kwargs): - """Initialize a Rebuild Container RF with ObjClass Write object.""" - super().__init__(*args, **kwargs) - self.daos_cmd = None - def test_container_redundancy_factor_oclass_enforcement(self): """Jira ID: DAOS-6267: Verify that a container can be created with and enforces @@ -42,7 +37,6 @@ def test_container_redundancy_factor_oclass_enforcement(self): :avocado: tags=vm :avocado: tags=container :avocado: tags=container_rf,cont_rf_oclass_enforcement - :avocado: tags=test_container_redundancy_factor_oclass_enforcement + :avocado: tags=ContRfEnforce,test_container_redundancy_factor_oclass_enforcement """ - self.mode = "cont_rf_enforcement" - self.execute_cont_rf_test() + self.execute_cont_rf_test("cont_rf_enforcement") diff --git a/src/tests/ftest/control/ms_resilience.py b/src/tests/ftest/control/ms_resilience.py index 45c03e909c4..8c646a40dbc 100644 --- a/src/tests/ftest/control/ms_resilience.py +++ b/src/tests/ftest/control/ms_resilience.py @@ -73,13 +73,13 @@ def create_pool(self): self.log.info("*** creating pool") self.pool.create() - self.log.info("Pool UUID %s on server group: %s", self.pool.uuid, self.server_group) + self.log.info("%s on server group: %s", str(self.pool), self.server_group) # Verify that the pool persisted. while not self.find_pool(self.pool.uuid): # Occasionally the pool may not be found # immediately after creation if the read # is serviced by a non-leader replica. - self.log.info("Pool %s not found yet.", self.pool.uuid) + self.log.info("%s not found yet.", str(self.pool)) time.sleep(1) self.log.info("Found pool in system.") diff --git a/src/tests/ftest/dfuse/bash.py b/src/tests/ftest/dfuse/bash.py index e220eecd13f..df43cb85e84 100644 --- a/src/tests/ftest/dfuse/bash.py +++ b/src/tests/ftest/dfuse/bash.py @@ -69,7 +69,7 @@ def run_bashcmd(self, il_lib=None, compatible_mode=False): # perform test for multiple containers. for count in range(cont_count): self.add_container(self.pool) - mount_dir = f"/tmp/{self.pool.uuid}_daos_dfuse{count}" + mount_dir = "/tmp/daos_dfuse_{}_{}".format(self.pool.identifier, count) self.start_dfuse( self.hostlist_clients, self.pool, self.container, mount_dir=mount_dir) if il_lib is not None: diff --git a/src/tests/ftest/io/macsio_test.py b/src/tests/ftest/io/macsio_test.py index 01daf179712..d4a074db8d4 100644 --- a/src/tests/ftest/io/macsio_test.py +++ b/src/tests/ftest/io/macsio_test.py @@ -33,20 +33,21 @@ def test_macsio(self): processes = self.params.get("processes", "/run/macsio/*", len(self.hostlist_clients)) # Create a pool - self.add_pool() - self.pool.display_pool_daos_space() + pool = self.get_pool() + pool.display_pool_daos_space() # Create a container - self.add_container(self.pool) + container = self.get_container(pool) # Run macsio self.log.info("Running MACSio") status = self.macsio.check_results( self.run_macsio( - self.pool.uuid, list_to_str(self.pool.svc_ranks), processes, self.container.uuid), + pool, list_to_str(pool.svc_ranks), processes, container), self.hostlist_clients) - if status: - self.log.info("Test passed") + if not status: + self.fail("MACSio failed") + self.log.info("Test passed") def test_macsio_daos_vol(self): """JIRA ID: DAOS-4983. @@ -68,14 +69,14 @@ def test_macsio_daos_vol(self): processes = self.params.get("processes", "/run/macsio/*", len(self.hostlist_clients)) # Create a pool - self.add_pool() - self.pool.display_pool_daos_space() + pool = self.get_pool() + pool.display_pool_daos_space() # Create a container - self.add_container(self.pool) + container = self.get_container(pool) # Create dfuse mount point - self.start_dfuse(self.hostlist_clients, self.pool, self.container) + self.start_dfuse(self.hostlist_clients, pool, container) # VOL needs to run from a file system that supports xattr. Currently # nfs does not have this attribute so it was recommended to create and @@ -86,8 +87,9 @@ def test_macsio_daos_vol(self): self.log.info("Running MACSio with DAOS VOL connector") status = self.macsio.check_results( self.run_macsio( - self.pool.uuid, list_to_str(self.pool.svc_ranks), processes, self.container.uuid, + pool, list_to_str(pool.svc_ranks), processes, container, plugin_path), self.hostlist_clients) - if status: - self.log.info("Test passed") + if not status: + self.fail("MACSio failed") + self.log.info("Test passed") diff --git a/src/tests/ftest/io/parallel_io.py b/src/tests/ftest/io/parallel_io.py index d4dbcae06a7..8a377affbd6 100644 --- a/src/tests/ftest/io/parallel_io.py +++ b/src/tests/ftest/io/parallel_io.py @@ -170,7 +170,7 @@ def test_parallelio(self): self.dfuse.mount_dir.value + "/" + container_to_destroy, False) self.fail( "Fio was able to access destroyed container: {}".format( - self.container[0].uuid)) + str(self.container[0]))) except CommandFailure: self.log.info("fio failed as expected") # check dfuse is still running after attempting to access deleted container diff --git a/src/tests/ftest/nvme/pool_capacity.py b/src/tests/ftest/nvme/pool_capacity.py index 2f88d012092..0c45cc88c45 100644 --- a/src/tests/ftest/nvme/pool_capacity.py +++ b/src/tests/ftest/nvme/pool_capacity.py @@ -178,8 +178,7 @@ def run_test(self, num_pool=1): # Destroy the last num_pool pools created offset = loop_count * num_pool for index in range(offset, offset + num_pool): - display_string = "Pool {} space at the End".format( - self.pool[index].uuid) + display_string = "{} space at the End".format(str(self.pool[index])) self.pool[index].display_pool_daos_space(display_string) self.pool[index].destroy() diff --git a/src/tests/ftest/pool/destroy.py b/src/tests/ftest/pool/destroy.py index 14cfd855224..1bb14ef1768 100644 --- a/src/tests/ftest/pool/destroy.py +++ b/src/tests/ftest/pool/destroy.py @@ -98,7 +98,6 @@ def validate_pool_creation(self, hosts, scm_mount): self.log.info("Create a pool") self.add_pool(create=False) self.pool.create() - self.log.info("Pool UUID is %s", self.pool.uuid) # Check that the pool was created. self.assertTrue( @@ -472,7 +471,7 @@ def test_destroy_connected(self): self.log.info("Check if files still exist") self.assertTrue( self.pool.check_files(hostlist_servers, scm_mount), - "Pool UUID {} should not be removed when connected".format(self.pool.uuid)) + "{} should not be removed when connected".format(str(self.pool))) self.assertTrue( exception_detected, "No exception when deleting a connected pool") @@ -562,7 +561,7 @@ def test_destroy_with_containers(self): self.log.info("Check if files still exist") self.assertTrue( self.pool.check_files(hostlist_servers, scm_mount), - "Pool UUID {} should not be removed when containers exist".format(self.pool.uuid)) + "{} should not be removed when containers exist".format(str(self.pool))) self.assertTrue( exception_detected, "No exception when deleting a pool with containers") diff --git a/src/tests/ftest/pool/list_pools.py b/src/tests/ftest/pool/list_pools.py index b1a85a0125d..d13204fcd1c 100644 --- a/src/tests/ftest/pool/list_pools.py +++ b/src/tests/ftest/pool/list_pools.py @@ -34,35 +34,36 @@ def run_case(self, rank_lists, svcn=None): """ # Iterate rank lists to create pools. Store the created pool information # as a dictionary of pool UUID keys with a service replica list value. - self.pool = [] - expected_uuids = {} + pools = [] + expected_uuids_labels = {} for rank_list in rank_lists: - self.pool.append(self.get_pool(target_list=rank_list, svcn=svcn)) - expected_uuids[self.pool[-1].uuid.lower()] = self.pool[-1].svc_ranks + pools.append(self.get_pool(target_list=rank_list, svcn=svcn)) + uuid_label = (pools[-1].uuid.lower(), pools[-1].label.value) + expected_uuids_labels[uuid_label] = pools[-1].svc_ranks # Verify the 'dmg pool info' command lists the correct created pool # information. The DmgCommand.pool_info() method returns the command # output as a dictionary of pool UUID keys with service replica list # values. - detected_uuids = {} + detected_uuids_labels = {} try: for data in self.get_dmg_command().get_pool_list_all(): - detected_uuids[data["uuid"]] = data["svc_reps"] + uuid_label = (data["uuid"], data["label"]) + detected_uuids_labels[uuid_label] = data["svc_reps"] except KeyError as error: self.fail("Error parsing dmg pool list output: {}".format(error)) - self.log.info("Expected pool info: %s", str(expected_uuids)) - self.log.info("Detected pool info: %s", str(detected_uuids)) + self.log.info("Expected pool info: %s", str(expected_uuids_labels)) + self.log.info("Detected pool info: %s", str(detected_uuids_labels)) # Destroy all the pools - if self.destroy_pools(self.pool): + if self.destroy_pools(pools): self.fail("Error destroying pools") - self.pool = [] # Compare the expected and detected pool information self.assertEqual( - expected_uuids, detected_uuids, - "dmg pool info does not list all expected pool UUIDs and their " + expected_uuids_labels, detected_uuids_labels, + "dmg pool info does not list all expected pool UUIDs, labels, and their " "service replicas") def test_list_pools(self): @@ -76,7 +77,7 @@ def test_list_pools(self): :avocado: tags=all,full_regression :avocado: tags=vm :avocado: tags=pool - :avocado: tags=list_pools,test_list_pools + :avocado: tags=ListPoolsTest,test_list_pools """ ranks = list(range(len(self.hostlist_servers))) diff --git a/src/tests/ftest/pool/query_attribute.py b/src/tests/ftest/pool/query_attribute.py index 988e1d18d1b..63f4e241bff 100644 --- a/src/tests/ftest/pool/query_attribute.py +++ b/src/tests/ftest/pool/query_attribute.py @@ -41,7 +41,7 @@ def test_query_attr(self): :avocado: tags=all,full_regression :avocado: tags=vm :avocado: tags=pool,pool_query,daos_cmd - :avocado: tags=pool_query_attr,test_query_attr + :avocado: tags=QueryAttributeTest,test_query_attr """ errors = [] daos_cmd = self.get_daos_command() @@ -52,8 +52,9 @@ def test_query_attr(self): # Call daos pool query, obtain pool UUID and SCM size, and compare # against those used when creating the pool. - query_result = daos_cmd.pool_query(pool=self.pool.uuid) + query_result = daos_cmd.pool_query(pool=self.pool.identifier) actual_uuid = query_result["response"]["uuid"] + actual_label = query_result["response"]["label"] actual_size = query_result["response"]["tier_stats"][0]["total"] actual_size_roundup = int(actual_size / 100000) * 100000 @@ -63,6 +64,12 @@ def test_query_attr(self): "Expected = {}; Actual = {}".format(expected_uuid, actual_uuid) errors.append(msg) + expected_label = self.pool.label.value + if expected_label != actual_label: + msg = "Unexpected label from daos pool query! " +\ + "Expected = {}; Actual = {}".format(expected_label, actual_label) + errors.append(msg) + if expected_size != actual_size_roundup: msg = "#Unexpected Total Storage Tier 0 size from daos pool " +\ "query! Expected = {}; Actual roundup = {}".format( @@ -82,11 +89,11 @@ def test_query_attr(self): sample_attrs.append(sample_attr) sample_vals.append(sample_val) daos_cmd.pool_set_attr( - pool=self.pool.uuid, attr=sample_attr, value=sample_val) + pool=self.pool.identifier, attr=sample_attr, value=sample_val) expected_attrs.append(sample_attr) # List the attribute names and compare against those set. - attrs = daos_cmd.pool_list_attrs(pool=self.pool.uuid) + attrs = daos_cmd.pool_list_attrs(pool=self.pool.identifier) for attr in attrs["response"]: actual_attrs.append(attr) @@ -102,7 +109,7 @@ def test_query_attr(self): # Get each attribute's value and compare against those set. for idx in range(5): output = daos_cmd.pool_get_attr( - pool=self.pool.uuid, attr=sample_attrs[idx]) + pool=self.pool.identifier, attr=sample_attrs[idx]) actual_val = base64.b64decode(output["response"]["value"]).decode() if sample_vals[idx] != actual_val: msg = "Unexpected attribute value! " +\ diff --git a/src/tests/ftest/rebuild/basic.py b/src/tests/ftest/rebuild/basic.py index 2d7b0e723c1..6e25fe3cd46 100644 --- a/src/tests/ftest/rebuild/basic.py +++ b/src/tests/ftest/rebuild/basic.py @@ -24,7 +24,6 @@ def run_rebuild_test(self, pool_quantity): # Get the test parameters pools = [] self.container = [] - daos_cmd = self.get_daos_command() for _ in range(pool_quantity): pools.append(self.get_pool(create=False)) self.container.append(self.get_container(pools[-1], create=False)) @@ -102,8 +101,7 @@ def run_rebuild_test(self, pool_quantity): # Verify the data after rebuild for index, pool in enumerate(pools): - daos_cmd.container_set_prop( - pool=pool.uuid, cont=self.container[index].uuid, prop="status", value="healthy") + self.container[index].set_prop(prop="status", value="healthy") if self.container[index].object_qty.value != 0: self.assertTrue( self.container[index].read_objects(), "Data verification error after rebuild") diff --git a/src/tests/ftest/rebuild/cascading_failures.py b/src/tests/ftest/rebuild/cascading_failures.py index a17c6c6ffac..8bfaf32cdab 100644 --- a/src/tests/ftest/rebuild/cascading_failures.py +++ b/src/tests/ftest/rebuild/cascading_failures.py @@ -3,7 +3,6 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from daos_utils import DaosCommand from rebuild_test_base import RebuildTestBase @@ -17,7 +16,6 @@ def __init__(self, *args, **kwargs): """Initialize a CascadingFailures object.""" super().__init__(*args, **kwargs) self.mode = None - self.daos_cmd = None def create_test_container(self): """Create a container and write objects.""" @@ -66,13 +64,11 @@ def start_rebuild(self): def execute_during_rebuild(self): """Execute test steps during rebuild.""" - self.daos_cmd = DaosCommand(self.bin) if self.mode == "cascading": # Exclude the second rank from the pool during rebuild self.server_managers[0].stop_ranks([self.inputs.rank.value[1]], self.d_log) - self.daos_cmd.container_set_prop( - pool=self.pool.uuid, cont=self.container.uuid, prop="status", value="healthy") + self.container.set_prop(prop="status", value="healthy") # Populate the container with additional data during rebuild self.container.write_objects(obj_class=self.inputs.object_class.value) diff --git a/src/tests/ftest/rebuild/container_rf.py b/src/tests/ftest/rebuild/container_rf.py index 51037212556..eaa0010f224 100644 --- a/src/tests/ftest/rebuild/container_rf.py +++ b/src/tests/ftest/rebuild/container_rf.py @@ -39,5 +39,4 @@ def test_rebuild_with_container_rf(self): :avocado: tags=container,rebuild,container_rf :avocado: tags=RbldContRfTest,test_rebuild_with_container_rf """ - self.mode = "cont_rf_with_rebuild" - self.execute_cont_rf_test() + self.execute_cont_rf_test("cont_rf_with_rebuild") diff --git a/src/tests/ftest/rebuild/no_cap.py b/src/tests/ftest/rebuild/no_cap.py index ff320837dc1..d56f806eda9 100644 --- a/src/tests/ftest/rebuild/no_cap.py +++ b/src/tests/ftest/rebuild/no_cap.py @@ -117,8 +117,7 @@ def test_rebuild_no_capacity(self): # query the pool before rebuild self.log.info("....Pool query after filling, written_pload=%s", written_pload) self.pool.set_query_data() - self.log.info( - "..Pool %s query data: %s\n", self.pool.uuid, self.pool.query_data) + self.log.info("..%s query data: %s\n", str(self.pool), self.pool.query_data) # Start rebuild rank = 1 diff --git a/src/tests/ftest/rebuild/pool_destroy_race.py b/src/tests/ftest/rebuild/pool_destroy_race.py index 4bc732c801f..8428d2e4681 100644 --- a/src/tests/ftest/rebuild/pool_destroy_race.py +++ b/src/tests/ftest/rebuild/pool_destroy_race.py @@ -33,7 +33,7 @@ def test_pool_destroy_with_io(self): :avocado: tags=all,pr :avocado: tags=hw,medium,md_on_ssd :avocado: tags=pool,rebuild,ior - :avocado: tags=pooldestroywithio,test_pool_destroy_with_io + :avocado: tags=RbldPoolDestroyWithIO,test_pool_destroy_with_io """ # set params targets = self.server_managers[0].get_config_value("targets") @@ -71,9 +71,9 @@ def test_pool_destroy_with_io(self): self.pool.wait_for_rebuild_to_start(interval=1) rebuild_state = self.pool.get_rebuild_state(True) - self.log.info("Pool %s rebuild status:%s", self.pool.uuid, rebuild_state) + self.log.info("%s rebuild status:%s", str(self.pool), rebuild_state) - self.log.info("Destroy pool %s while rebuild is %s", self.pool.uuid, rebuild_state) + self.log.info("Destroy %s while rebuild is %s", str(self.pool), rebuild_state) self.pool.destroy() self.container = None diff --git a/src/tests/ftest/rebuild/read_array.py b/src/tests/ftest/rebuild/read_array.py index 6b72bf45bbd..b2c0f33d2af 100644 --- a/src/tests/ftest/rebuild/read_array.py +++ b/src/tests/ftest/rebuild/read_array.py @@ -3,7 +3,6 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from daos_utils import DaosCommand from general_utils import DaosTestError from rebuild_test_base import RebuildTestBase @@ -14,16 +13,9 @@ class RbldReadArrayTest(RebuildTestBase): :avocado: recursive """ - def __init__(self, *args, **kwargs): - """Initialize a RbldReadArrayTest object.""" - super().__init__(*args, **kwargs) - self.daos_cmd = None - def execute_during_rebuild(self): """Read the objects during rebuild.""" - self.daos_cmd = DaosCommand(self.bin) - self.daos_cmd.container_set_prop( - pool=self.pool.uuid, cont=self.container.uuid, prop="status", value="healthy") + self.container.set_prop(prop="status", value="healthy") message = "Reading the array objects during rebuild" self.log.info(message) diff --git a/src/tests/ftest/rebuild/with_io.py b/src/tests/ftest/rebuild/with_io.py index 7e7a1e623d4..eaf60ad7ac3 100644 --- a/src/tests/ftest/rebuild/with_io.py +++ b/src/tests/ftest/rebuild/with_io.py @@ -4,7 +4,6 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ from apricot import TestWithServers -from daos_utils import DaosCommand class RbldWithIO(TestWithServers): @@ -62,7 +61,7 @@ def test_rebuild_with_io(self): # Write data to the container for 30 seconds self.log.info( "Wrote %s bytes to container %s", - self.container.execute_io(30, rank, obj_class), self.container.uuid) + self.container.execute_io(30, rank, obj_class), str(self.container)) # Determine how many objects will need to be rebuilt self.container.get_target_rank_lists(" prior to rebuild") @@ -73,14 +72,12 @@ def test_rebuild_with_io(self): # Wait for recovery to start self.pool.wait_for_rebuild_to_start() - daos_cmd = DaosCommand(self.bin) - daos_cmd.container_set_prop( - pool=self.pool.uuid, cont=self.container.uuid, prop="status", value="healthy") + self.container.set_prop(prop="status", value="healthy") # Write data to the container for another 30 seconds self.log.info( "Wrote an additional %s bytes to container %s", - self.container.execute_io(30), self.container.uuid) + self.container.execute_io(30), str(self.container)) # Wait for recovery to complete self.pool.wait_for_rebuild_to_end() diff --git a/src/tests/ftest/scrubber/basic.py b/src/tests/ftest/scrubber/basic.py index f1fbabf984c..13ec352c360 100644 --- a/src/tests/ftest/scrubber/basic.py +++ b/src/tests/ftest/scrubber/basic.py @@ -34,7 +34,7 @@ def run_scrubber_basic(self, pool_prop=None, cont_prop=None): for test in transfer_block_size: self.ior_cmd.transfer_size.update(test[0]) self.ior_cmd.block_size.update(test[1]) - status = self.run_ior_and_check_scruber_status(pool=self.pool, cont=self.container) + status = self.run_ior_and_check_scrubber_status(pool=self.pool, cont=self.container) if status is False: self.log.info("-------Test Failed-------") self.log.info("---No metrics value change----") @@ -54,7 +54,7 @@ def test_scrubber_disabled_during_pool_creation(self): :avocado: tags=all,pr,daily_regression :avocado: tags=hw,medium :avocado: tags=scrubber - :avocado: tags=test_scrubber_disabled_during_pool_creation + :avocado: tags=TestWithScrubberBasic,test_scrubber_disabled_during_pool_creation """ self.run_scrubber_basic(None, None) @@ -71,7 +71,7 @@ def test_scrubber_enabled_during_pool_creation(self): :avocado: tags=all,pr,daily_regression :avocado: tags=hw,medium :avocado: tags=scrubber - :avocado: tags=test_scrubber_enabled_during_pool_creation + :avocado: tags=TestWithScrubberBasic,test_scrubber_enabled_during_pool_creation """ pool_prop = self.params.get("properties", '/run/pool/*') diff --git a/src/tests/ftest/scrubber/csum_fault.py b/src/tests/ftest/scrubber/csum_fault.py index 4003f6a13f1..2107ff0e1c4 100644 --- a/src/tests/ftest/scrubber/csum_fault.py +++ b/src/tests/ftest/scrubber/csum_fault.py @@ -43,7 +43,7 @@ def test_scrubber_csum_fault(self): for test in transfer_block_size: self.ior_cmd.transfer_size.update(test[0]) self.ior_cmd.block_size.update(test[1]) - self.run_ior_and_check_scruber_status(pool=self.pool, cont=self.container) + self.run_ior_and_check_scrubber_status(pool=self.pool, cont=self.container) start_time = 0 finish_time = 0 poll_status = False diff --git a/src/tests/ftest/scrubber/target_auto_eviction.py b/src/tests/ftest/scrubber/target_auto_eviction.py index 5a2ecef8327..183c80a66e9 100644 --- a/src/tests/ftest/scrubber/target_auto_eviction.py +++ b/src/tests/ftest/scrubber/target_auto_eviction.py @@ -30,13 +30,13 @@ def test_scrubber_ssd_auto_eviction(self): initial_metrics = {} final_metrics = {} self.create_pool_cont_with_scrubber(pool_prop=pool_prop, cont_prop=cont_prop) - self.dmg_cmd.pool_query(self.pool.identifier) + self.pool.query() initial_metrics = self.scrubber.get_scrub_corrupt_metrics() - self.run_ior_and_check_scruber_status(pool=self.pool, cont=self.container) + self.run_ior_and_check_scrubber_status(pool=self.pool, cont=self.container) # Wait for a minute for the scrubber to take action and evict target # after corruption threshold reached. time.sleep(60) - self.dmg_cmd.pool_query(self.pool.identifier) + self.pool.query() final_metrics = self.scrubber.get_scrub_corrupt_metrics() status = self.verify_scrubber_metrics_value(initial_metrics, final_metrics) if status is False: diff --git a/src/tests/ftest/server/metadata.py b/src/tests/ftest/server/metadata.py index 18cf9edf70f..654a88ecbd3 100644 --- a/src/tests/ftest/server/metadata.py +++ b/src/tests/ftest/server/metadata.py @@ -90,7 +90,7 @@ def pre_tear_down(self): def create_pool(self): """Create a pool and display the svc ranks.""" self.add_pool() - self.log.info("Created pool %s: svc ranks:", self.pool.uuid) + self.log.info("Created %s: svc ranks:", str(self.pool)) for index, rank in enumerate(self.pool.svc_ranks): self.log.info("[%d]: %d", index, rank) @@ -196,7 +196,7 @@ def test_metadata_fillup(self): :avocado: tags=all,full_regression :avocado: tags=hw,large :avocado: tags=server,metadata - :avocado: tags=metadata_fillup,test_metadata_fillup + :avocado: tags=ObjectMetadata,test_metadata_fillup """ self.create_pool() @@ -293,7 +293,7 @@ def test_metadata_addremove(self): :avocado: tags=all,full_regression :avocado: tags=hw,large :avocado: tags=server,metadata,nvme - :avocado: tags=metadata_compact,metadata_addremove,test_metadata_addremove + :avocado: tags=ObjectMetadata,test_metadata_addremove """ self.create_pool() self.container = [] @@ -363,7 +363,7 @@ def test_metadata_server_restart(self): :avocado: tags=all,full_regression :avocado: tags=hw,large :avocado: tags=server,metadata,nvme - :avocado: tags=metadata_ior,test_metadata_server_restart + :avocado: tags=ObjectMetadata,test_metadata_server_restart """ self.create_pool() files_per_thread = 400 @@ -437,7 +437,7 @@ def test_container_removal_after_der_nospace(self): :avocado: tags=all,full_regression :avocado: tags=hw,large :avocado: tags=server,metadata,nvme - :avocado: tags=metadata_der_nospace,der_nospace,test_container_removal_after_der_nospace + :avocado: tags=ObjectMetadata,test_container_removal_after_der_nospace """ self.create_pool() diff --git a/src/tests/ftest/util/apricot/apricot/test.py b/src/tests/ftest/util/apricot/apricot/test.py index 78a75dae36a..49cab3a1ce3 100644 --- a/src/tests/ftest/util/apricot/apricot/test.py +++ b/src/tests/ftest/util/apricot/apricot/test.py @@ -1838,7 +1838,7 @@ def get_container(self, pool, namespace=None, create=True, daos_command=None, ** return container - def add_container(self, pool, namespace=None, create=True): + def add_container(self, pool, namespace=None, create=True, **kwargs): """Add a container to the test case. This method defines the common test container creation sequence. @@ -1849,8 +1849,11 @@ def add_container(self, pool, namespace=None, create=True): the test yaml file. Defaults to None. create (bool, optional): should the container be created. Defaults to True. + kwargs (dict): name/value of attributes for which to call update(value, name). + See TestContainer for available attributes. """ - self.container = self.get_container(pool=pool, namespace=namespace, create=create) + self.container = self.get_container( + pool=pool, namespace=namespace, create=create, **kwargs) def add_container_qty(self, quantity, pool, namespace=None, create=True): """Add multiple containers to the test case. diff --git a/src/tests/ftest/util/container_rf_test_base.py b/src/tests/ftest/util/container_rf_test_base.py index cf89ea6355c..567f7d3f984 100644 --- a/src/tests/ftest/util/container_rf_test_base.py +++ b/src/tests/ftest/util/container_rf_test_base.py @@ -5,7 +5,6 @@ """ import re -from daos_utils import DaosCommand from general_utils import DaosTestError from rebuild_test_base import RebuildTestBase @@ -16,12 +15,6 @@ class ContRedundancyFactor(RebuildTestBase): :avocado: recursive """ - def __init__(self, *args, **kwargs): - """Initialize a CascadingFailures object.""" - super().__init__(*args, **kwargs) - self.mode = None - self.daos_cmd = None - def create_test_container(self): """Create a container and write objects.""" self.log.info( @@ -61,8 +54,7 @@ def verify_cont_rf_healthstatus(self, expected_rf, expected_health): actual_rf = None actual_health = None - cont_props = self.daos_cmd.container_get_prop( - pool=self.pool.uuid, cont=self.container.uuid, properties=["rd_fac", "status"]) + cont_props = self.container.get_prop(properties=["rd_fac", "status"]) for cont_prop in cont_props["response"]: if cont_prop["name"] == "rd_fac": actual_rf = cont_prop["value"] @@ -144,16 +136,16 @@ def create_test_container_and_write_obj(self, negative_test=False): self.fail("#Negative test, container redundancy factor " "test failed, return error RC: -1003 not found") - def execute_cont_rf_test(self, create_container=True): + def execute_cont_rf_test(self, mode, create_container=True): """Execute the rebuild test steps for container rd_fac test. Args: + mode (str): "cont_rf_with_rebuild" or "cont_rf_enforcement" create_container (bool, optional): should the test create a container. Defaults to True. """ # Get the test params and var self.setup_test_pool() - self.daos_cmd = DaosCommand(self.bin) if create_container: self.setup_test_container() oclass = self.inputs.object_class.value @@ -171,7 +163,7 @@ def execute_cont_rf_test(self, create_container=True): self.create_test_pool() # Create a container and write objects self.create_test_container_and_write_obj(negative_test) - if self.mode == "cont_rf_with_rebuild": + if mode == "cont_rf_with_rebuild": num_of_ranks = len(self.inputs.rank.value) if num_of_ranks > rf_num: expect_cont_status = "UNCLEAN" @@ -196,7 +188,7 @@ def execute_cont_rf_test(self, create_container=True): self.log.info("==>(7)Check for container data if the container is healthy.") self.verify_container_data() self.log.info("Test passed") - elif self.mode == "cont_rf_enforcement": + elif mode == "cont_rf_enforcement": self.log.info("Container rd_fac test passed") else: self.fail("#Unsupported container_rf test mode") diff --git a/src/tests/ftest/util/data_mover_test_base.py b/src/tests/ftest/util/data_mover_test_base.py index 4a839bb587f..5603521f025 100644 --- a/src/tests/ftest/util/data_mover_test_base.py +++ b/src/tests/ftest/util/data_mover_test_base.py @@ -363,7 +363,7 @@ def get_cont(self, pool, cont): """ # Query the container for existence and to get the uuid from a label - query_response = self.daos_cmd.container_query(pool=pool.uuid, cont=cont)['response'] + query_response = self.daos_cmd.container_query(pool=pool.identifier, cont=cont)['response'] cont_uuid = query_response['container_uuid'] cont_label = query_response.get('container_label') @@ -417,8 +417,7 @@ def dataset_gen(self, cont, num_objs, num_dkeys, num_akeys_single, list: a list of DaosObj created. """ - self.log.info("Creating dataset in %s/%s", - str(cont.pool.uuid), str(cont.uuid)) + self.log.info("Creating dataset in %s/%s", str(cont.pool), str(cont)) cont.open() @@ -488,8 +487,7 @@ def dataset_verify(self, obj_list, cont, num_objs, num_dkeys, akey_extents (list): varying number of akey extents to iterate. """ - self.log.info("Verifying dataset in %s/%s", - str(cont.pool.uuid), str(cont.uuid)) + self.log.info("Verifying dataset in %s/%s", str(cont.pool), str(cont)) cont.open() diff --git a/src/tests/ftest/util/dfuse_utils.py b/src/tests/ftest/util/dfuse_utils.py index c9f10bb9250..2a47ad8e2f2 100644 --- a/src/tests/ftest/util/dfuse_utils.py +++ b/src/tests/ftest/util/dfuse_utils.py @@ -423,7 +423,7 @@ def start_dfuse(test, dfuse, pool=None, container=None, **params): if pool: params['pool'] = pool.identifier if container: - params['cont'] = container.uuid + params['cont'] = container.identifier if params: dfuse.update_params(**params) diff --git a/src/tests/ftest/util/macsio_test_base.py b/src/tests/ftest/util/macsio_test_base.py index e7519a51eb3..f1f888dcaf0 100644 --- a/src/tests/ftest/util/macsio_test_base.py +++ b/src/tests/ftest/util/macsio_test_base.py @@ -42,17 +42,17 @@ def get_macsio_command(self): return macsio - def run_macsio(self, pool_uuid, pool_svcl, processes, cont_uuid=None, plugin=None, slots=None): + def run_macsio(self, pool, pool_svcl, processes, cont=None, plugin=None, slots=None): """Run the macsio test. Parameters for the macsio command are obtained from the test yaml file, including the path to the macsio executable. Args: - pool_uuid (str): pool uuid + pool (TestPool): pool to run with pool_svcl (str): pool service replica processes (int): total number of processes to use to run macsio - cont_uuid (str, optional): container uuid. Defaults to None. + cont (TestContainer, optional): container to run with. Defaults to None. plugin (str, optional): plugin path to use with DAOS VOL connector slots (int, optional): slots per host to specify in the hostfile. Defaults to None. @@ -64,9 +64,9 @@ def run_macsio(self, pool_uuid, pool_svcl, processes, cont_uuid=None, plugin=Non """ # Update the MACSio pool and container info before gathering manager # environment information to ensure they are included. - self.macsio.daos_pool = pool_uuid + self.macsio.daos_pool = pool.identifier self.macsio.daos_svcl = pool_svcl - self.macsio.daos_cont = cont_uuid + self.macsio.daos_cont = cont.identifier # Setup the job manager to run the macsio command env = self.macsio.env.copy() @@ -88,6 +88,6 @@ def run_macsio(self, pool_uuid, pool_svcl, processes, cont_uuid=None, plugin=Non except CommandFailure as error: self.log.error("MACSio Failed: %s", str(error)) - self.fail("MACSio Failed.\n") + self.fail("MACSio Failed") return result diff --git a/src/tests/ftest/util/osa_utils.py b/src/tests/ftest/util/osa_utils.py index 0b9c9002809..e327f1398c7 100644 --- a/src/tests/ftest/util/osa_utils.py +++ b/src/tests/ftest/util/osa_utils.py @@ -445,7 +445,7 @@ def ior_thread(self, pool, oclass, test, flags, single_cont_read=True, fail_on_w except CommandFailure as err_msg: self.out_queue.put(err_msg) self.assert_on_exception() - job_manager.job.dfs_cont.update(self.container.uuid) + job_manager.job.dfs_cont.update(self.container.identifier) self.ior_cmd.transfer_size.update(test[2]) self.ior_cmd.block_size.update(test[3]) self.ior_cmd.flags.update(flags) @@ -486,7 +486,7 @@ def run_mdtest_thread(self, oclass="RP_2G1"): if create_container == 1: self.container.create() job_manager = self.get_mdtest_job_manager_command(self.manager) - job_manager.job.dfs_cont.update(self.container.uuid) + job_manager.job.dfs_cont.update(self.container.identifier) # Add a thread for these IOR arguments process = threading.Thread(target=self.execute_mdtest) # Launch the MDtest thread diff --git a/src/tests/ftest/util/performance_test_base.py b/src/tests/ftest/util/performance_test_base.py index bf7a24907ef..1225712b294 100644 --- a/src/tests/ftest/util/performance_test_base.py +++ b/src/tests/ftest/util/performance_test_base.py @@ -217,9 +217,7 @@ def verify_system_status(self, pool=None, container=None): if pool: funcs.append(pool.set_query_data) if container: - funcs.append( - lambda: self.log.info( - self.daos_cmd.container_query(container.pool.identifier, container.uuid))) + funcs.append(container.query) first_error = None for func in funcs: diff --git a/src/tests/ftest/util/rebuild_test_base.py b/src/tests/ftest/util/rebuild_test_base.py index bdb31ea9807..8ac5d042a82 100644 --- a/src/tests/ftest/util/rebuild_test_base.py +++ b/src/tests/ftest/util/rebuild_test_base.py @@ -5,7 +5,6 @@ """ from apricot import TestWithServers from command_utils_base import BasicParameter, ObjectWithParameters -from daos_utils import DaosCommand class RebuildTestParams(ObjectWithParameters): @@ -33,7 +32,6 @@ def __init__(self, *args, **kwargs): self.server_count = 0 self.info_checks = None self.rebuild_checks = None - self.daos_cmd = None def setUp(self): """Set up each test case.""" @@ -160,7 +158,6 @@ def execute_rebuild_test(self, create_container=True): """ # Get the test params self.setup_test_pool() - self.daos_cmd = DaosCommand(self.bin) if create_container: self.setup_test_container() @@ -183,11 +180,7 @@ def execute_rebuild_test(self, create_container=True): self.pool.wait_for_rebuild_to_end(1) # clear container status for the RF issue - self.daos_cmd.container_set_prop( - pool=self.pool.uuid, - cont=self.container.uuid, - prop="status", - value="healthy") + self.container.set_prop(prop="status", value="healthy") # Refresh local pool and container self.pool.check_pool_info() diff --git a/src/tests/ftest/util/scrubber_test_base.py b/src/tests/ftest/util/scrubber_test_base.py index 7e938662e9f..3cf99e2a887 100644 --- a/src/tests/ftest/util/scrubber_test_base.py +++ b/src/tests/ftest/util/scrubber_test_base.py @@ -23,14 +23,9 @@ def __init__(self, *args, **kwargs): def setUp(self): """Set up each test case.""" super().setUp() - self.scrubber = ScrubberUtils(self.get_dmg_command(), - self.server_managers[0].hosts) - self.daos_cmd = self.get_daos_command() - self.dmg_cmd = self.get_dmg_command() + self.scrubber = ScrubberUtils(self.get_dmg_command(), self.server_managers[0].hosts) self.pool = None self.container = None - self.initial_metrics = {} - self.final_metrics = {} def verify_scrubber_metrics_value(self, initial_metrics, final_metrics): """Compare the initial metrics value to final value after IO data. @@ -73,47 +68,40 @@ def create_pool_cont_with_scrubber(self, pool_prop=None, cont_prop=None): # Testing scenario : Create a pool and container without properties # and update them at runtime. if pool_prop is None: - self.add_pool(create=False, connect=False) - self.pool.properties.value = pool_prop - self.pool.create() - self.pool.connect() + # Create without properties and set at runtime below + self.add_pool(properties=None) else: + # Create with properties self.add_pool() - self.add_container(pool=self.pool, create=False) if pool_prop is None: pool_prop = "scrub:timed,scrub-freq:1" - if cont_prop is None: - cont_prop = "cksum:crc16" for prop_val in pool_prop.split(","): if prop_val is not None: value = prop_val.split(":") self.pool.set_property(value[0], value[1]) - self.container.properties.value = cont_prop - self.container.create() - values = "Pool : {} Container: {}".format(self.pool, self.container) - self.log.info(values) + if cont_prop is None: + cont_prop = "cksum:crc16" + self.add_container(pool=self.pool, properties=cont_prop) - def run_ior_and_check_scruber_status(self, pool, cont, fail_on_warning=True): + def run_ior_and_check_scrubber_status(self, pool, cont): """Run IOR and get scrubber metrics Args: pool (object): Pool object cont (object): Container object within the pool. - fail_on_warning (bool, optional): [description]. Defaults to True. Returns: - status(bool) : True (Scrubber working), False(Scrubber not working) + bool: True (Scrubber working), False(Scrubber not working) """ - status = False - self.initial_metrics = self.scrubber.get_csum_total_metrics() + initial_metrics = self.scrubber.get_csum_total_metrics() self.pool = pool self.container = cont # Print the pool properties - result = self.dmg_cmd.pool_get_prop(self.pool.uuid, "scrub") + result = self.pool.get_prop("scrub") self.log.info("Pool Properties") self.log.info("===============") self.log.info(result) - result = self.daos_cmd.container_get_prop(self.pool.uuid, self.container.uuid) + result = self.container.get_prop() self.log.info("Container Properties") self.log.info("===============") self.log.info(result) @@ -122,12 +110,11 @@ def run_ior_and_check_scruber_status(self, pool, cont, fail_on_warning=True): process = threading.Thread(target=self.run_ior_with_pool, kwargs={"create_pool": True, "create_cont": False, - "fail_on_warning": fail_on_warning}) + "fail_on_warning": True}) # Launch the IOR thread process.start() # Wait for the thread to finish process.join() - self.final_metrics = self.scrubber.get_csum_total_metrics() + final_metrics = self.scrubber.get_csum_total_metrics() # Just make sure scrubber is working here. - status = self.verify_scrubber_metrics_value(self.initial_metrics, self.final_metrics) - return status + return self.verify_scrubber_metrics_value(initial_metrics, final_metrics) diff --git a/src/tests/ftest/util/test_utils_container.py b/src/tests/ftest/util/test_utils_container.py index 3eedc1158d4..5c36cfee711 100644 --- a/src/tests/ftest/util/test_utils_container.py +++ b/src/tests/ftest/util/test_utils_container.py @@ -100,7 +100,7 @@ def write_record(self, container, akey, dkey, data, rank=None, "Error writing {}data (dkey={}, akey={}, data={}) to " "container {}: {}".format( "array " if isinstance(data, list) else "", dkey, akey, - data, container.uuid, error)) from error + data, str(container), error)) from error def write_object(self, container, record_qty, akey_size, dkey_size, data_size, rank=None, obj_class=None, data_array_size=0): @@ -178,7 +178,7 @@ def read_record(self, container, akey, dkey, data_size, data_array_size=0, "Error reading {}data (dkey={}, akey={}, size={}) from " "container {}: {}".format( "array " if data_array_size > 0 else "", dkey, akey, - data_size, container.uuid, error)) from error + data_size, str(container), error)) from error return [data[:-1] for data in read_data] \ if data_array_size > 0 else read_data.value diff --git a/src/tests/ftest/util/vol_test_base.py b/src/tests/ftest/util/vol_test_base.py index b45bcf62082..9a06c34b2fa 100644 --- a/src/tests/ftest/util/vol_test_base.py +++ b/src/tests/ftest/util/vol_test_base.py @@ -29,14 +29,14 @@ def run_test(self, job_manager, plugin_path, test_repo): client_processes = self.params.get("client_processes") # create pool, container and dfuse mount - self.add_pool(connect=False) - self.add_container(self.pool) + pool = self.get_pool(connect=False) + container = self.get_container(pool) # VOL needs to run from a file system that supports xattr. # Currently nfs does not have this attribute so it was recommended # to create a dfuse dir and run vol tests from there. # create dfuse container - self.start_dfuse(self.hostlist_clients, self.pool, self.container) + self.start_dfuse(self.hostlist_clients, pool, container) # Assign the test to run job_manager.job = ExecutableCommand( @@ -44,8 +44,8 @@ def run_test(self, job_manager, plugin_path, test_repo): check_results=["FAILED", "stderr"]) env = EnvironmentVariables() - env["DAOS_POOL"] = "{}".format(self.pool.uuid) - env["DAOS_CONT"] = "{}".format(self.container.uuid) + env["DAOS_POOL"] = pool.identifier + env["DAOS_CONT"] = container.identifier env["HDF5_VOL_CONNECTOR"] = "daos" env["HDF5_PLUGIN_PATH"] = "{}".format(plugin_path) job_manager.assign_hosts(self.hostlist_clients) From 37e683358119eaf8ab4af6d62796716f8af8952a Mon Sep 17 00:00:00 2001 From: Dalton Bohning Date: Mon, 27 Nov 2023 17:38:30 +0000 Subject: [PATCH 2/8] fixes Test-tag: ContainerQueryAttributeTest ManagementServiceResilience test_bashcmd ParallelIo NvmePoolCapacity DestroyTests ListPoolsTest RbldBasic RbldBasic RbldPoolDestroyWithIO RbldReadArrayTest RbldWithIO ObjectMetadata performance,-manual container,pr DaosVolBigIO DaosVol MacsioTest TestWithScrubberBasic TestWithScrubberFault TestWithScrubberTargetEviction dfuse datamover test_container_redundancy_factor_oclass_enforcement RbldContRfTest osa RbldCascadingFailures RbldDeleteObjects RbldReadArrayTest TestScrubberEvictWithAggregation TestScrubberEvictWithRebuild TestScrubberEvictWithSnapshot Skip-unit-tests: true Skip-fault-injection-test: true Required-githooks: true Signed-off-by: Dalton Bohning --- src/tests/ftest/scrubber/aggregation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/ftest/scrubber/aggregation.py b/src/tests/ftest/scrubber/aggregation.py index 9d3e7828fde..f248c9efa4e 100644 --- a/src/tests/ftest/scrubber/aggregation.py +++ b/src/tests/ftest/scrubber/aggregation.py @@ -44,7 +44,7 @@ def test_target_eviction_during_aggregation(self): # Now enable the scrubber on the pool. self.pool.set_prop(properties="scrub:timed,scrub-freq:1,scrub-thresh:3") initial_metrics = self.scrubber.get_scrub_corrupt_metrics() - self.run_ior_and_check_scruber_status(pool=self.pool, cont=self.container) + self.run_ior_and_check_scrubber_status(pool=self.pool, cont=self.container) # We want both aggregation and scrubber tasks # to run in parallel during this time. start_time = 0 From e256bbbf289080e85ff713bdfd5a2ea62be00b32 Mon Sep 17 00:00:00 2001 From: Dalton Bohning Date: Mon, 27 Nov 2023 17:55:56 +0000 Subject: [PATCH 3/8] fixes Test-tag: ContainerQueryAttributeTest ManagementServiceResilience test_bashcmd ParallelIo NvmePoolCapacity DestroyTests ListPoolsTest RbldBasic RbldBasic RbldPoolDestroyWithIO RbldReadArrayTest RbldWithIO ObjectMetadata performance,-manual container,pr DaosVolBigIO DaosVol MacsioTest TestWithScrubberBasic TestWithScrubberFault TestWithScrubberTargetEviction dfuse datamover test_container_redundancy_factor_oclass_enforcement RbldContRfTest osa RbldCascadingFailures RbldDeleteObjects RbldReadArrayTest TestScrubberEvictWithAggregation TestScrubberEvictWithRebuild TestScrubberEvictWithSnapshot TestWithScrubberFreq Skip-unit-tests: true Skip-fault-injection-test: true Required-githooks: true Signed-off-by: Dalton Bohning --- src/tests/ftest/scrubber/frequency.py | 2 +- src/tests/ftest/scrubber/rebuild.py | 2 +- src/tests/ftest/scrubber/snapshot.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tests/ftest/scrubber/frequency.py b/src/tests/ftest/scrubber/frequency.py index 1db16550300..d2f089069e8 100644 --- a/src/tests/ftest/scrubber/frequency.py +++ b/src/tests/ftest/scrubber/frequency.py @@ -36,7 +36,7 @@ def test_objects_scrubbed_properly(self): cont_prop = self.params.get("properties", '/run/container/*') self.create_pool_cont_with_scrubber(pool_prop=pool_prop, cont_prop=cont_prop) # Run IOR and gather the total scrubbed metrics information. - self.run_ior_and_check_scruber_status(pool=self.pool, cont=self.container) + self.run_ior_and_check_scrubber_status(pool=self.pool, cont=self.container) # Wait for 5 minutes to get first scrubber bytes scrubbed metrics. # NOTE: This value could change depending on the IOR data (objects created) self.log.info("Sleeping for 5 minutes pool property set to scrub:timed") diff --git a/src/tests/ftest/scrubber/rebuild.py b/src/tests/ftest/scrubber/rebuild.py index e2e079f15ad..0feaa228dcc 100644 --- a/src/tests/ftest/scrubber/rebuild.py +++ b/src/tests/ftest/scrubber/rebuild.py @@ -33,7 +33,7 @@ def test_target_eviction_during_rebuild(self): self.create_pool_cont_with_scrubber(pool_prop=pool_prop, cont_prop=cont_prop) self.pool.query() initial_metrics = self.scrubber.get_scrub_corrupt_metrics() - self.run_ior_and_check_scruber_status(pool=self.pool, cont=self.container) + self.run_ior_and_check_scrubber_status(pool=self.pool, cont=self.container) # Exclude Rank 5 to start the rebuild operation. self.pool.exclude("5") # Wait for a minute for the scrubber to take action and evict target diff --git a/src/tests/ftest/scrubber/snapshot.py b/src/tests/ftest/scrubber/snapshot.py index ace7e147cae..cac713b7a28 100644 --- a/src/tests/ftest/scrubber/snapshot.py +++ b/src/tests/ftest/scrubber/snapshot.py @@ -33,7 +33,7 @@ def test_target_eviction_during_snapshot(self): self.create_pool_cont_with_scrubber(pool_prop=pool_prop, cont_prop=cont_prop) self.pool.query() initial_metrics = self.scrubber.get_scrub_corrupt_metrics() - self.run_ior_and_check_scruber_status(pool=self.pool, cont=self.container) + self.run_ior_and_check_scrubber_status(pool=self.pool, cont=self.container) # Wait for a minute for the scrubber to take action and evict target # after corruption threshold reached. # Take a snap-shot after 15 seconds while the csum faults are injected. From 9ed7bac33b3c26de5b6395c60c67699680c2a09b Mon Sep 17 00:00:00 2001 From: Dalton Bohning Date: Mon, 29 Jan 2024 21:29:18 +0000 Subject: [PATCH 4/8] fixes after merge Test-tag: ContainerQueryAttributeTest ManagementServiceResilience test_bashcmd ParallelIo NvmePoolCapacity DestroyTests ListPoolsTest RbldBasic RbldBasic RbldPoolDestroyWithIO RbldReadArrayTest RbldWithIO ObjectMetadata performance,-manual container,pr DaosVolBigIO DaosVol MacsioTest TestWithScrubberBasic TestWithScrubberFault TestWithScrubberTargetEviction dfuse datamover test_container_redundancy_factor_oclass_enforcement RbldContRfTest osa RbldCascadingFailures RbldDeleteObjects RbldReadArrayTest TestScrubberEvictWithAggregation TestScrubberEvictWithRebuild TestScrubberEvictWithSnapshot TestWithScrubberFreq Skip-unit-tests: true Skip-fault-injection-test: true Required-githooks: true Signed-off-by: Dalton Bohning --- src/tests/ftest/container/rf_enforcement.py | 3 +-- src/tests/ftest/util/container_rf_test_base.py | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/tests/ftest/container/rf_enforcement.py b/src/tests/ftest/container/rf_enforcement.py index d38b2862cd5..c96f0db1af7 100644 --- a/src/tests/ftest/container/rf_enforcement.py +++ b/src/tests/ftest/container/rf_enforcement.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2019-2023 Intel Corporation. + (C) Copyright 2019-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -36,7 +36,6 @@ def test_container_redundancy_factor_oclass_enforcement(self): :avocado: tags=all,full_regression :avocado: tags=vm :avocado: tags=container - :avocado: tags=container_rf,cont_rf_oclass_enforcement :avocado: tags=ContRfEnforce,test_container_redundancy_factor_oclass_enforcement """ self.execute_cont_rf_test(mode="cont_rf_enforcement") diff --git a/src/tests/ftest/util/container_rf_test_base.py b/src/tests/ftest/util/container_rf_test_base.py index 000730efcf9..37dbc8a397f 100644 --- a/src/tests/ftest/util/container_rf_test_base.py +++ b/src/tests/ftest/util/container_rf_test_base.py @@ -131,7 +131,6 @@ def execute_cont_rf_test(self, create_container=True, mode=None): """Execute the rebuild test steps for container rd_fac test. Args: - mode (str): "cont_rf_with_rebuild" or "cont_rf_enforcement" create_container (bool, optional): should the test create a container. Defaults to True. mode (str): either "cont_rf_with_rebuild" or "cont_rf_enforcement" From d842b5064b9f2a147814e1c826b787754ca633bf Mon Sep 17 00:00:00 2001 From: Dalton Bohning Date: Thu, 22 Feb 2024 18:30:44 +0000 Subject: [PATCH 5/8] fix bad merge Test-tag: ContainerQueryAttributeTest ManagementServiceResilience ParallelIo NvmePoolCapacity DestroyTests ListPoolsTest RbldBasic RbldBasic RbldPoolDestroyWithIO RbldReadArrayTest RbldWithIO ObjectMetadata performance,-manual container,pr DaosVolBigIO DaosVol MacsioTest TestWithScrubberBasic TestWithScrubberFault TestWithScrubberTargetEviction dfuse datamover test_container_redundancy_factor_oclass_enforcement RbldContRfTest osa RbldCascadingFailures RbldDeleteObjects RbldReadArrayTest TestScrubberEvictWithAggregation TestScrubberEvictWithRebuild TestScrubberEvictWithSnapshot TestWithScrubberFreq Skip-unit-tests: true Skip-fault-injection-test: true Required-githooks: true Signed-off-by: Dalton Bohning --- src/tests/ftest/util/scrubber_test_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/ftest/util/scrubber_test_base.py b/src/tests/ftest/util/scrubber_test_base.py index 3cf99e2a887..f3a828e366d 100644 --- a/src/tests/ftest/util/scrubber_test_base.py +++ b/src/tests/ftest/util/scrubber_test_base.py @@ -1,5 +1,5 @@ """ -(C) Copyright 2021-2023 Intel Corporation. +(C) Copyright 2021-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -74,7 +74,7 @@ def create_pool_cont_with_scrubber(self, pool_prop=None, cont_prop=None): # Create with properties self.add_pool() if pool_prop is None: - pool_prop = "scrub:timed,scrub-freq:1" + pool_prop = "scrub:timed,scrub_freq:1" for prop_val in pool_prop.split(","): if prop_val is not None: value = prop_val.split(":") From 394a6e85cea513ca6cd256b4d522b0357be01e90 Mon Sep 17 00:00:00 2001 From: Dalton Bohning Date: Mon, 4 Mar 2024 18:59:43 +0000 Subject: [PATCH 6/8] update test-tag re-evaluate Test-tag since there have been many updates in master and some of the original changes are no longer in this PR. Test-tag: ContRfEnforce ManagementServiceResilience MacsioTest ParallelIo NvmePoolCapacity DestroyTests ListPoolsTest QueryAttributeTest RbldBasic RbldCascadingFailures RbldNoCapacity RbldPoolDestroyWithIO RbldReadArrayTest RbldWithIO scrubber ObjectMetadata DmvrDstCreate DmvrObjSmallTest DmvrPreserveProps RbldDeleteObjects DaosVolBigIO DaosVol osa Skip-unit-tests: true Skip-fault-injection-test: true Required-githooks: true Signed-off-by: Dalton Bohning From e922295d794726cbb3bdf0c6ed8bfe57b79e7771 Mon Sep 17 00:00:00 2001 From: Dalton Bohning Date: Tue, 5 Mar 2024 16:14:24 +0000 Subject: [PATCH 7/8] fix test Test-tag: ContRfEnforce ManagementServiceResilience MacsioTest ParallelIo NvmePoolCapacity DestroyTests ListPoolsTest QueryAttributeTest RbldBasic RbldCascadingFailures RbldNoCapacity RbldPoolDestroyWithIO RbldReadArrayTest RbldWithIO scrubber ObjectMetadata DmvrDstCreate DmvrObjSmallTest DmvrPreserveProps RbldDeleteObjects DaosVolBigIO DaosVol osa Skip-unit-tests: true Skip-fault-injection-test: true Required-githooks: true Signed-off-by: Dalton Bohning --- src/tests/ftest/pool/query_attribute.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/tests/ftest/pool/query_attribute.py b/src/tests/ftest/pool/query_attribute.py index 63f4e241bff..bed2c4984f8 100644 --- a/src/tests/ftest/pool/query_attribute.py +++ b/src/tests/ftest/pool/query_attribute.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2020-2023 Intel Corporation. + (C) Copyright 2020-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -54,7 +54,6 @@ def test_query_attr(self): # against those used when creating the pool. query_result = daos_cmd.pool_query(pool=self.pool.identifier) actual_uuid = query_result["response"]["uuid"] - actual_label = query_result["response"]["label"] actual_size = query_result["response"]["tier_stats"][0]["total"] actual_size_roundup = int(actual_size / 100000) * 100000 @@ -64,12 +63,6 @@ def test_query_attr(self): "Expected = {}; Actual = {}".format(expected_uuid, actual_uuid) errors.append(msg) - expected_label = self.pool.label.value - if expected_label != actual_label: - msg = "Unexpected label from daos pool query! " +\ - "Expected = {}; Actual = {}".format(expected_label, actual_label) - errors.append(msg) - if expected_size != actual_size_roundup: msg = "#Unexpected Total Storage Tier 0 size from daos pool " +\ "query! Expected = {}; Actual roundup = {}".format( From f0e1d01d4237f245fcb4ac2d6efe0cdb6ffb028d Mon Sep 17 00:00:00 2001 From: Dalton Bohning Date: Tue, 10 Sep 2024 15:16:52 +0000 Subject: [PATCH 8/8] fix macsio_test Features: ContRfEnforce DestroyTests ListPoolsTest MacsioTest ManagementServiceResilience NvmePoolCapacity ObjectMetadata ParallelIo QueryAttributeTest RbldCascadingFailures RbldNoCapacity RbldPoolDestroyWithIO RbldReadArrayTest RbldWithIO TestScrubberEvictWithAggregation TestScrubberEvictWithRebuild TestScrubberEvictWithSnapshot TestWithScrubberBasic TestWithScrubberFault TestWithScrubberFreq TestWithScrubberTargetEviction scrubber DmvrDstCreate DmvrObjSmallTest DmvrPreserveProps RbldDeleteObjects DaosVolBigIO DaosVol osa Skip-unit-tests: true Skip-fault-injection-test: true Required-githooks: true Signed-off-by: Dalton Bohning --- src/tests/ftest/io/macsio_test.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/tests/ftest/io/macsio_test.py b/src/tests/ftest/io/macsio_test.py index 83353a1cd22..348017968d6 100644 --- a/src/tests/ftest/io/macsio_test.py +++ b/src/tests/ftest/io/macsio_test.py @@ -18,12 +18,11 @@ class MacsioTest(TestWithServers): :avocado: recursive """ - def get_macsio_command(self, pool, pool_svcl, cont): + def _get_macsio_command(self, pool, cont): """Get the MacsioCommand object. Args: pool (TestPool): pool object - pool_svcl (str): pool service replica cont (TestContainer): container object Returns: @@ -40,7 +39,7 @@ def get_macsio_command(self, pool, pool_svcl, cont): # Update the MACSio pool and container info before gathering manager # environment information to ensure they are included. macsio.daos_pool = pool.identifier - macsio.daos_svcl = pool_svcl + macsio.daos_svcl = list_to_str(pool.svc_ranks) macsio.daos_cont = cont.identifier return macsio @@ -115,7 +114,7 @@ def test_macsio(self): # Run macsio self.log_step("Running MACSio") - macsio = self.get_macsio_command(pool.uuid, list_to_str(pool.svc_ranks), container.uuid) + macsio = self._get_macsio_command(pool, container) result = self.run_macsio(macsio, self.hostlist_clients, processes) if not macsio.check_results(result, self.hostlist_clients): self.fail("MACSio failed") @@ -159,7 +158,7 @@ def test_macsio_daos_vol(self): # Run macsio self.log_step("Running MACSio with DAOS VOL connector") - macsio = self.get_macsio_command(pool, list_to_str(pool.svc_ranks), container) + macsio = self._get_macsio_command(pool, container) result = self.run_macsio( macsio, self.hostlist_clients, processes, plugin_path, working_dir=dfuse.mount_dir.value)