diff --git a/tests/rptest/tests/topic_creation_test.py b/tests/rptest/tests/topic_creation_test.py index 24de7a76f1d6..1c904722851c 100644 --- a/tests/rptest/tests/topic_creation_test.py +++ b/tests/rptest/tests/topic_creation_test.py @@ -741,360 +741,3 @@ def metadata_consistent(): return True wait_until(metadata_consistent, 45, backoff_sec=2) - - -class CreateTopicUpgradeTest(RedpandaTest): - def __init__(self, test_context): - si_settings = SISettings(test_context, - cloud_storage_enable_remote_write=False, - cloud_storage_enable_remote_read=False) - self._s3_bucket = si_settings.cloud_storage_bucket - - super(CreateTopicUpgradeTest, self).__init__(test_context=test_context, - num_brokers=3, - si_settings=si_settings) - self.installer = self.redpanda._installer - self.rpk = RpkTool(self.redpanda) - - # This test starts the Redpanda service inline (see 'install_and_start') at the beginning - # of the test body. By default, in the Azure CDT env, the service startup - # logic attempts to set the azure specific cluster configs. - # However, these did not exist prior to v23.1 and the test would fail - # before it can be skipped. - def setUp(self): - pass - - def install_and_start(self): - self.installer.install( - self.redpanda.nodes, - (22, 2), - ) - self.redpanda.start() - - def _populate_tiered_storage_topic(self, topic_name, local_retention): - # Write 3x the local retention, then wait for local storage to be - # trimmed back accordingly. - bytes = local_retention * 3 - msg_size = 131072 - msg_count = bytes // msg_size - for _ in range(0, max(msg_count, 4)): - self.rpk.produce(topic_name, "key", "b" * msg_size) - - wait_for_local_storage_truncate(self.redpanda, - topic=topic_name, - target_bytes=local_retention) - - @cluster(num_nodes=3) - @matrix(cloud_storage_type=get_cloud_storage_type( - applies_only_on=[CloudStorageType.S3])) - def test_cloud_storage_sticky_enablement_v22_2_to_v22_3( - self, cloud_storage_type): - """ - In Redpanda 22.3, the cluster defaults for cloud storage change - from being applied at runtime to being sticky at creation time, - or at upgrade time. - """ - self.install_and_start() - - # Switch on tiered storage using cluster properties, not topic properties - self.redpanda.set_cluster_config( - { - 'cloud_storage_enable_remote_read': True, - 'cloud_storage_enable_remote_write': True - }, - # This shouldn't require a restart, but it does in Redpanda < 22.3 - True) - - topic = "test-topic" - self.rpk.create_topic(topic) - described = self.rpk.describe_topic_configs(topic) - assert described['redpanda.remote.write'] == ('true', 'DEFAULT_CONFIG') - assert described['redpanda.remote.read'] == ('true', 'DEFAULT_CONFIG') - - # Upgrade to Redpanda latest - self.installer.install(self.redpanda.nodes, (22, 3)) - self.redpanda.restart_nodes(self.redpanda.nodes) - - # Wait for properties migration to run - self.redpanda.await_feature("cloud_retention", - active=True, - timeout_sec=30) - - self.logger.info( - f"Config status after upgrade: {self.redpanda._admin.get_cluster_config_status()}" - ) - - # Properties should still be set, but migrated to topic-level - described = self.rpk.describe_topic_configs(topic) - assert described['redpanda.remote.write'] == ('true', 'DEFAULT_CONFIG') - assert described['redpanda.remote.read'] == ('true', 'DEFAULT_CONFIG') - - # A new topic picks up these properties too - self.rpk.create_topic("created-after-enabled") - described = self.rpk.describe_topic_configs("created-after-enabled") - assert described['redpanda.remote.write'] == ('true', 'DEFAULT_CONFIG') - assert described['redpanda.remote.read'] == ('true', 'DEFAULT_CONFIG') - - # Switching off cluster defaults shoudln't affect existing topic - self.redpanda.set_cluster_config( - { - 'cloud_storage_enable_remote_read': False, - 'cloud_storage_enable_remote_write': False - }, False) - described = self.rpk.describe_topic_configs(topic) - assert described['redpanda.remote.write'] == ('true', - 'DYNAMIC_TOPIC_CONFIG') - assert described['redpanda.remote.read'] == ('true', - 'DYNAMIC_TOPIC_CONFIG') - - # A newly created topic should have tiered storage switched off - self.rpk.create_topic("created-after-disabled") - described = self.rpk.describe_topic_configs("created-after-disabled") - assert described['redpanda.remote.write'] == ('false', - 'DEFAULT_CONFIG') - assert described['redpanda.remote.read'] == ('false', 'DEFAULT_CONFIG') - - # Nothing in this test guarantees that partition manifests will be uploaded. - self.redpanda.si_settings.set_expected_damage({"ntpr_no_manifest"}) - - @cluster(num_nodes=3) - @matrix(cloud_storage_type=get_cloud_storage_type( - applies_only_on=[CloudStorageType.S3])) - def test_retention_config_on_upgrade_from_v22_2_to_v22_3( - self, cloud_storage_type): - self.install_and_start() - - self.rpk.create_topic("test-topic-with-retention", - config={"retention.bytes": 10000}) - - local_retention = 10000 - self.rpk.create_topic( - "test-si-topic-with-retention", - config={ - "retention.bytes": str(local_retention), - "redpanda.remote.write": "true", - "redpanda.remote.read": "true", - # Small segment.bytes so that we can readily - # get data into S3 by writing small amounts. - "segment.bytes": 1000000 - }) - - # Write a few megabytes of data, enough to spill to S3. This will be used - # later for checking that deletion behavior is correct on legacy topics. - self._populate_tiered_storage_topic("test-si-topic-with-retention", - local_retention) - - # This topic is like "test-si-topic-with-retention", but instead - # of settings its properties at creation time, set them via - # alter messages: these follow a different code path in - # kafka and controller, and are serialized using different structures - # than the structures used in topic creation. - self.rpk.create_topic("test-si-topic-with-retention-altered") - self.rpk.alter_topic_config("test-si-topic-with-retention-altered", - "retention.bytes", 10000) - self.rpk.alter_topic_config("test-si-topic-with-retention-altered", - "redpanda.remote.write", "true") - - # Check our alter operations applied properly - # (see https://github.com/redpanda-data/redpanda/issues/6772) - after_alter = self.rpk.describe_topic_configs( - "test-si-topic-with-retention-altered") - assert after_alter['redpanda.remote.write'][0] == 'true' - assert after_alter['retention.bytes'][0] == '10000' - - # TODO: this test currently exercises 22.2 (serde) encoding to - # 22.3 (newer serde) encoding. It should be extended to also - # cover the case of creating topics in version 22.1, upgrading - # to 22.2, and then to 22.3, to check the ADL encoding of messages - # in the controller log. - - self.installer.install( - self.redpanda.nodes, - (22, 3), - ) - - self.redpanda.restart_nodes(self.redpanda.nodes) - - # Wait for any migration steps to complete - self.redpanda.await_feature("cloud_retention", - active=True, - timeout_sec=30) - - non_si_configs = self.rpk.describe_topic_configs( - "test-topic-with-retention") - si_configs = self.rpk.describe_topic_configs( - "test-si-topic-with-retention") - si_altered_configs = self.rpk.describe_topic_configs( - "test-si-topic-with-retention-altered") - - # "test-topic-with-retention" did not have remote write enabled - # at the time of the upgrade, so retention.* configs are preserved - # and retention.local.target.* configs are set to their default values, - # but ignored - self.logger.debug(f"Checking config {non_si_configs}") - assert non_si_configs["retention.bytes"] == ("10000", - "DYNAMIC_TOPIC_CONFIG") - assert non_si_configs["retention.ms"][1] == "DEFAULT_CONFIG" - - assert non_si_configs["retention.local.target.bytes"] == ( - "-1", "DEFAULT_CONFIG") - assert non_si_configs["retention.local.target.ms"][ - 1] == "DEFAULT_CONFIG" - - # 'test-si-topic-with-retention' was enabled from remote write - # at the time of the upgrade, so retention.local.target.* configs - # should be initialised from retention.* and retention.* configs should - # be disabled. - for conf in (si_configs, si_altered_configs): - self.logger.debug(f"Checking config: {conf}") - assert conf['redpanda.remote.write'][0] == 'true' - assert conf["retention.bytes"][0] == "-1" - assert conf["retention.ms"][0] == "-1" - - assert conf["retention.local.target.bytes"] == ( - "10000", "DYNAMIC_TOPIC_CONFIG") - assert conf["retention.local.target.ms"][1] == "DEFAULT_CONFIG" - assert conf["redpanda.remote.delete"][0] == "false" - - # After upgrade, newly created topics should have remote.delete - # enabled by default, and interpret assignments to retention properties - # literally (no mapping of retention -> retention.local) - for (new_topic_name, - enable_si) in [("test-topic-post-upgrade-nosi", False), - ("test-topic-post-upgrade-si", True)]: - - segment_bytes = 1000000 - local_retention = segment_bytes * 2 - retention_bytes = segment_bytes * 10 - create_config = { - "retention.bytes": retention_bytes, - "retention.local.target.bytes": local_retention, - "segment.bytes": segment_bytes - } - if enable_si: - create_config['redpanda.remote.write'] = 'true' - create_config['redpanda.remote.read'] = 'true' - - self.rpk.create_topic(new_topic_name, config=create_config) - new_config = self.rpk.describe_topic_configs(new_topic_name) - assert new_config["redpanda.remote.delete"][0] == "true" - assert new_config["retention.bytes"] == (str(retention_bytes), - "DYNAMIC_TOPIC_CONFIG") - assert new_config["retention.ms"][1] == "DEFAULT_CONFIG" - assert new_config["retention.local.target.ms"][ - 1] == "DEFAULT_CONFIG" - assert new_config["retention.local.target.bytes"] == ( - str(local_retention), "DYNAMIC_TOPIC_CONFIG") - if enable_si: - assert new_config['redpanda.remote.write'][0] == "true" - assert new_config['redpanda.remote.read'][0] == "true" - - # The remote.delete property is applied irrespective of whether - # the topic is initially tiered storage enabled. - assert new_config['redpanda.remote.delete'][0] == "true" - - if enable_si: - self._populate_tiered_storage_topic(new_topic_name, - local_retention) - - # A newly created tiered storage topic should have its data deleted - # in S3 when the topic is deleted - self._delete_tiered_storage_topic("test-topic-post-upgrade-si", True) - - # Ensure that the `redpanda.remote.delete==false` configuration is - # really taking effect, by deleting a legacy topic and ensuring data is - # left behind in S3 - self._delete_tiered_storage_topic("test-si-topic-with-retention", - False) - - def _delete_tiered_storage_topic(self, topic_name: str, - expect_s3_deletion: bool): - self.logger.debug(f"Deleting {topic_name} and checking S3 result") - - before_objects = set( - o.key - for o in self.cloud_storage_client.list_objects(self._s3_bucket) - if topic_name in o.key) - - # Test is meaningless if there were no objects to start with - assert len(before_objects) > 0 - - self.rpk.delete_topic(topic_name) - - def is_empty(): - return sum(1 for o in self.cloud_storage_client.list_objects( - self._s3_bucket) if topic_name in o.key) == 0 - - if expect_s3_deletion: - wait_until(is_empty, timeout_sec=10, backoff_sec=1) - else: - # When we expect objects to remain, require that they remain for - # at least some time, to avoid false-passing if they were deleted with - # some small delay. - sleep(10) - - after_objects = set( - o.key - for o in self.cloud_storage_client.list_objects(self._s3_bucket) - if topic_name in o.key) - deleted_objects = before_objects - after_objects - if expect_s3_deletion: - assert deleted_objects - else: - self.logger.debug( - f"deleted objects (should be empty): {deleted_objects}") - assert len(deleted_objects) == 0 - - @cluster(num_nodes=3) - @matrix(cloud_storage_type=get_cloud_storage_type( - applies_only_on=[CloudStorageType.S3])) - def test_retention_upgrade_with_cluster_remote_write( - self, cloud_storage_type): - """ - Validate how the cluster-wide cloud_storage_enable_remote_write - is handled on upgrades from <=22.2 - """ - self.install_and_start() - - self.redpanda.set_cluster_config( - {"cloud_storage_enable_remote_write": "true"}, expect_restart=True) - - self.rpk.create_topic("test-topic-with-remote-write", - config={"retention.bytes": 10000}) - - self.rpk.create_topic("test-topic-without-remote-write", - config={ - "retention.bytes": 10000, - "redpanda.remote.write": "false" - }) - - self.installer.install( - self.redpanda.nodes, - (22, 3), - ) - - self.redpanda.restart_nodes(self.redpanda.nodes) - - # Wait for any migration steps to complete - self.redpanda.await_feature('cloud_retention', - active=True, - timeout_sec=30) - - # Because legacy Redpanda treated cloud_storage_enable_remote_write as - # an override to the topic property (even if the topic remote.write was explicitly - # set to false), our two topics should end up with identical config: both - # with SI enabled and with their retention settings transposed. - for topic in [ - "test-topic-with-remote-write", - "test-topic-without-remote-write" - ]: - config = self.rpk.describe_topic_configs(topic) - assert config["retention.ms"] == ("-1", "DYNAMIC_TOPIC_CONFIG") - assert config["retention.bytes"] == ("-1", "DYNAMIC_TOPIC_CONFIG") - assert config["retention.local.target.bytes"] == ( - "10000", "DYNAMIC_TOPIC_CONFIG") - assert config["retention.local.target.ms"][1] == "DEFAULT_CONFIG" - - # The ntp manifest may not have been uploaded yet, but that's fine - # since this test focuses on the topic config change. - self.redpanda.si_settings.set_expected_damage({"ntpr_no_manifest"})