Skip to content

Commit

Permalink
Merge pull request redpanda-data#16799 from mmaslankaprv/drop-old-upg…
Browse files Browse the repository at this point in the history 
…rade-tests

tests: dropped old upgrade tests which validated v22.2 version
  • Loading branch information
@mmaslankaprv
mmaslankaprv authored Mar 1, 2024
2 parents bdd7e92 + 5f7ce53 commit 9b7ff60
Showing 1 changed file with 0 additions and 357 deletions.
357 changes: 0 additions & 357 deletions tests/rptest/tests/topic_creation_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -741,360 +741,3 @@ def metadata_consistent():
return True

wait_until(metadata_consistent, 45, backoff_sec=2)


class CreateTopicUpgradeTest(RedpandaTest):
def __init__(self, test_context):
si_settings = SISettings(test_context,
cloud_storage_enable_remote_write=False,
cloud_storage_enable_remote_read=False)
self._s3_bucket = si_settings.cloud_storage_bucket

super(CreateTopicUpgradeTest, self).__init__(test_context=test_context,
num_brokers=3,
si_settings=si_settings)
self.installer = self.redpanda._installer
self.rpk = RpkTool(self.redpanda)

# This test starts the Redpanda service inline (see 'install_and_start') at the beginning
# of the test body. By default, in the Azure CDT env, the service startup
# logic attempts to set the azure specific cluster configs.
# However, these did not exist prior to v23.1 and the test would fail
# before it can be skipped.
def setUp(self):
pass

def install_and_start(self):
self.installer.install(
self.redpanda.nodes,
(22, 2),
)
self.redpanda.start()

def _populate_tiered_storage_topic(self, topic_name, local_retention):
# Write 3x the local retention, then wait for local storage to be
# trimmed back accordingly.
bytes = local_retention * 3
msg_size = 131072
msg_count = bytes // msg_size
for _ in range(0, max(msg_count, 4)):
self.rpk.produce(topic_name, "key", "b" * msg_size)

wait_for_local_storage_truncate(self.redpanda,
topic=topic_name,
target_bytes=local_retention)

@cluster(num_nodes=3)
@matrix(cloud_storage_type=get_cloud_storage_type(
applies_only_on=[CloudStorageType.S3]))
def test_cloud_storage_sticky_enablement_v22_2_to_v22_3(
self, cloud_storage_type):
"""
In Redpanda 22.3, the cluster defaults for cloud storage change
from being applied at runtime to being sticky at creation time,
or at upgrade time.
"""
self.install_and_start()

# Switch on tiered storage using cluster properties, not topic properties
self.redpanda.set_cluster_config(
{
'cloud_storage_enable_remote_read': True,
'cloud_storage_enable_remote_write': True
},
# This shouldn't require a restart, but it does in Redpanda < 22.3
True)

topic = "test-topic"
self.rpk.create_topic(topic)
described = self.rpk.describe_topic_configs(topic)
assert described['redpanda.remote.write'] == ('true', 'DEFAULT_CONFIG')
assert described['redpanda.remote.read'] == ('true', 'DEFAULT_CONFIG')

# Upgrade to Redpanda latest
self.installer.install(self.redpanda.nodes, (22, 3))
self.redpanda.restart_nodes(self.redpanda.nodes)

# Wait for properties migration to run
self.redpanda.await_feature("cloud_retention",
active=True,
timeout_sec=30)

self.logger.info(
f"Config status after upgrade: {self.redpanda._admin.get_cluster_config_status()}"
)

# Properties should still be set, but migrated to topic-level
described = self.rpk.describe_topic_configs(topic)
assert described['redpanda.remote.write'] == ('true', 'DEFAULT_CONFIG')
assert described['redpanda.remote.read'] == ('true', 'DEFAULT_CONFIG')

# A new topic picks up these properties too
self.rpk.create_topic("created-after-enabled")
described = self.rpk.describe_topic_configs("created-after-enabled")
assert described['redpanda.remote.write'] == ('true', 'DEFAULT_CONFIG')
assert described['redpanda.remote.read'] == ('true', 'DEFAULT_CONFIG')

# Switching off cluster defaults shoudln't affect existing topic
self.redpanda.set_cluster_config(
{
'cloud_storage_enable_remote_read': False,
'cloud_storage_enable_remote_write': False
}, False)
described = self.rpk.describe_topic_configs(topic)
assert described['redpanda.remote.write'] == ('true',
'DYNAMIC_TOPIC_CONFIG')
assert described['redpanda.remote.read'] == ('true',
'DYNAMIC_TOPIC_CONFIG')

# A newly created topic should have tiered storage switched off
self.rpk.create_topic("created-after-disabled")
described = self.rpk.describe_topic_configs("created-after-disabled")
assert described['redpanda.remote.write'] == ('false',
'DEFAULT_CONFIG')
assert described['redpanda.remote.read'] == ('false', 'DEFAULT_CONFIG')

# Nothing in this test guarantees that partition manifests will be uploaded.
self.redpanda.si_settings.set_expected_damage({"ntpr_no_manifest"})

@cluster(num_nodes=3)
@matrix(cloud_storage_type=get_cloud_storage_type(
applies_only_on=[CloudStorageType.S3]))
def test_retention_config_on_upgrade_from_v22_2_to_v22_3(
self, cloud_storage_type):
self.install_and_start()

self.rpk.create_topic("test-topic-with-retention",
config={"retention.bytes": 10000})

local_retention = 10000
self.rpk.create_topic(
"test-si-topic-with-retention",
config={
"retention.bytes": str(local_retention),
"redpanda.remote.write": "true",
"redpanda.remote.read": "true",
# Small segment.bytes so that we can readily
# get data into S3 by writing small amounts.
"segment.bytes": 1000000
})

# Write a few megabytes of data, enough to spill to S3. This will be used
# later for checking that deletion behavior is correct on legacy topics.
self._populate_tiered_storage_topic("test-si-topic-with-retention",
local_retention)

# This topic is like "test-si-topic-with-retention", but instead
# of settings its properties at creation time, set them via
# alter messages: these follow a different code path in
# kafka and controller, and are serialized using different structures
# than the structures used in topic creation.
self.rpk.create_topic("test-si-topic-with-retention-altered")
self.rpk.alter_topic_config("test-si-topic-with-retention-altered",
"retention.bytes", 10000)
self.rpk.alter_topic_config("test-si-topic-with-retention-altered",
"redpanda.remote.write", "true")

# Check our alter operations applied properly
# (see https://github.com/redpanda-data/redpanda/issues/6772)
after_alter = self.rpk.describe_topic_configs(
"test-si-topic-with-retention-altered")
assert after_alter['redpanda.remote.write'][0] == 'true'
assert after_alter['retention.bytes'][0] == '10000'

# TODO: this test currently exercises 22.2 (serde) encoding to
# 22.3 (newer serde) encoding. It should be extended to also
# cover the case of creating topics in version 22.1, upgrading
# to 22.2, and then to 22.3, to check the ADL encoding of messages
# in the controller log.

self.installer.install(
self.redpanda.nodes,
(22, 3),
)

self.redpanda.restart_nodes(self.redpanda.nodes)

# Wait for any migration steps to complete
self.redpanda.await_feature("cloud_retention",
active=True,
timeout_sec=30)

non_si_configs = self.rpk.describe_topic_configs(
"test-topic-with-retention")
si_configs = self.rpk.describe_topic_configs(
"test-si-topic-with-retention")
si_altered_configs = self.rpk.describe_topic_configs(
"test-si-topic-with-retention-altered")

# "test-topic-with-retention" did not have remote write enabled
# at the time of the upgrade, so retention.* configs are preserved
# and retention.local.target.* configs are set to their default values,
# but ignored
self.logger.debug(f"Checking config {non_si_configs}")
assert non_si_configs["retention.bytes"] == ("10000",
"DYNAMIC_TOPIC_CONFIG")
assert non_si_configs["retention.ms"][1] == "DEFAULT_CONFIG"

assert non_si_configs["retention.local.target.bytes"] == (
"-1", "DEFAULT_CONFIG")
assert non_si_configs["retention.local.target.ms"][
1] == "DEFAULT_CONFIG"

# 'test-si-topic-with-retention' was enabled from remote write
# at the time of the upgrade, so retention.local.target.* configs
# should be initialised from retention.* and retention.* configs should
# be disabled.
for conf in (si_configs, si_altered_configs):
self.logger.debug(f"Checking config: {conf}")
assert conf['redpanda.remote.write'][0] == 'true'
assert conf["retention.bytes"][0] == "-1"
assert conf["retention.ms"][0] == "-1"

assert conf["retention.local.target.bytes"] == (
"10000", "DYNAMIC_TOPIC_CONFIG")
assert conf["retention.local.target.ms"][1] == "DEFAULT_CONFIG"
assert conf["redpanda.remote.delete"][0] == "false"

# After upgrade, newly created topics should have remote.delete
# enabled by default, and interpret assignments to retention properties
# literally (no mapping of retention -> retention.local)
for (new_topic_name,
enable_si) in [("test-topic-post-upgrade-nosi", False),
("test-topic-post-upgrade-si", True)]:

segment_bytes = 1000000
local_retention = segment_bytes * 2
retention_bytes = segment_bytes * 10
create_config = {
"retention.bytes": retention_bytes,
"retention.local.target.bytes": local_retention,
"segment.bytes": segment_bytes
}
if enable_si:
create_config['redpanda.remote.write'] = 'true'
create_config['redpanda.remote.read'] = 'true'

self.rpk.create_topic(new_topic_name, config=create_config)
new_config = self.rpk.describe_topic_configs(new_topic_name)
assert new_config["redpanda.remote.delete"][0] == "true"
assert new_config["retention.bytes"] == (str(retention_bytes),
"DYNAMIC_TOPIC_CONFIG")
assert new_config["retention.ms"][1] == "DEFAULT_CONFIG"
assert new_config["retention.local.target.ms"][
1] == "DEFAULT_CONFIG"
assert new_config["retention.local.target.bytes"] == (
str(local_retention), "DYNAMIC_TOPIC_CONFIG")
if enable_si:
assert new_config['redpanda.remote.write'][0] == "true"
assert new_config['redpanda.remote.read'][0] == "true"

# The remote.delete property is applied irrespective of whether
# the topic is initially tiered storage enabled.
assert new_config['redpanda.remote.delete'][0] == "true"

if enable_si:
self._populate_tiered_storage_topic(new_topic_name,
local_retention)

# A newly created tiered storage topic should have its data deleted
# in S3 when the topic is deleted
self._delete_tiered_storage_topic("test-topic-post-upgrade-si", True)

# Ensure that the `redpanda.remote.delete==false` configuration is
# really taking effect, by deleting a legacy topic and ensuring data is
# left behind in S3
self._delete_tiered_storage_topic("test-si-topic-with-retention",
False)

def _delete_tiered_storage_topic(self, topic_name: str,
expect_s3_deletion: bool):
self.logger.debug(f"Deleting {topic_name} and checking S3 result")

before_objects = set(
o.key
for o in self.cloud_storage_client.list_objects(self._s3_bucket)
if topic_name in o.key)

# Test is meaningless if there were no objects to start with
assert len(before_objects) > 0

self.rpk.delete_topic(topic_name)

def is_empty():
return sum(1 for o in self.cloud_storage_client.list_objects(
self._s3_bucket) if topic_name in o.key) == 0

if expect_s3_deletion:
wait_until(is_empty, timeout_sec=10, backoff_sec=1)
else:
# When we expect objects to remain, require that they remain for
# at least some time, to avoid false-passing if they were deleted with
# some small delay.
sleep(10)

after_objects = set(
o.key
for o in self.cloud_storage_client.list_objects(self._s3_bucket)
if topic_name in o.key)
deleted_objects = before_objects - after_objects
if expect_s3_deletion:
assert deleted_objects
else:
self.logger.debug(
f"deleted objects (should be empty): {deleted_objects}")
assert len(deleted_objects) == 0

@cluster(num_nodes=3)
@matrix(cloud_storage_type=get_cloud_storage_type(
applies_only_on=[CloudStorageType.S3]))
def test_retention_upgrade_with_cluster_remote_write(
self, cloud_storage_type):
"""
Validate how the cluster-wide cloud_storage_enable_remote_write
is handled on upgrades from <=22.2
"""
self.install_and_start()

self.redpanda.set_cluster_config(
{"cloud_storage_enable_remote_write": "true"}, expect_restart=True)

self.rpk.create_topic("test-topic-with-remote-write",
config={"retention.bytes": 10000})

self.rpk.create_topic("test-topic-without-remote-write",
config={
"retention.bytes": 10000,
"redpanda.remote.write": "false"
})

self.installer.install(
self.redpanda.nodes,
(22, 3),
)

self.redpanda.restart_nodes(self.redpanda.nodes)

# Wait for any migration steps to complete
self.redpanda.await_feature('cloud_retention',
active=True,
timeout_sec=30)

# Because legacy Redpanda treated cloud_storage_enable_remote_write as
# an override to the topic property (even if the topic remote.write was explicitly
# set to false), our two topics should end up with identical config: both
# with SI enabled and with their retention settings transposed.
for topic in [
"test-topic-with-remote-write",
"test-topic-without-remote-write"
]:
config = self.rpk.describe_topic_configs(topic)
assert config["retention.ms"] == ("-1", "DYNAMIC_TOPIC_CONFIG")
assert config["retention.bytes"] == ("-1", "DYNAMIC_TOPIC_CONFIG")
assert config["retention.local.target.bytes"] == (
"10000", "DYNAMIC_TOPIC_CONFIG")
assert config["retention.local.target.ms"][1] == "DEFAULT_CONFIG"

# The ntp manifest may not have been uploaded yet, but that's fine
# since this test focuses on the topic config change.
self.redpanda.si_settings.set_expected_damage({"ntpr_no_manifest"})

0 comments on commit 9b7ff60

Please sign in to comment.