Skip to content

Commit

Permalink
tests: dropped old upgrade tests which validated v22.2 version
Browse files Browse the repository at this point in the history 
Dropped tests which validated upgrades between no longer supported
Redpanda versions (from v22.2.x to v22.3.x).

Fixes: #16356

Signed-off-by: Michal Maslanka <michal@redpanda.com>
  • Loading branch information
@mmaslankaprv
mmaslankaprv committed Feb 29, 2024
1 parent 182e420 commit 5f7ce53
Showing 1 changed file with 0 additions and 357 deletions.
357 changes: 0 additions & 357 deletions tests/rptest/tests/topic_creation_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -741,360 +741,3 @@ def metadata_consistent():
return True

wait_until(metadata_consistent, 45, backoff_sec=2)


class CreateTopicUpgradeTest(RedpandaTest):
def __init__(self, test_context):
si_settings = SISettings(test_context,
cloud_storage_enable_remote_write=False,
cloud_storage_enable_remote_read=False)
self._s3_bucket = si_settings.cloud_storage_bucket

super(CreateTopicUpgradeTest, self).__init__(test_context=test_context,
num_brokers=3,
si_settings=si_settings)
self.installer = self.redpanda._installer
self.rpk = RpkTool(self.redpanda)

# This test starts the Redpanda service inline (see 'install_and_start') at the beginning
# of the test body. By default, in the Azure CDT env, the service startup
# logic attempts to set the azure specific cluster configs.
# However, these did not exist prior to v23.1 and the test would fail
# before it can be skipped.
def setUp(self):
pass

def install_and_start(self):
self.installer.install(
self.redpanda.nodes,
(22, 2),
)
self.redpanda.start()

def _populate_tiered_storage_topic(self, topic_name, local_retention):
# Write 3x the local retention, then wait for local storage to be
# trimmed back accordingly.
bytes = local_retention * 3
msg_size = 131072
msg_count = bytes // msg_size
for _ in range(0, max(msg_count, 4)):
self.rpk.produce(topic_name, "key", "b" * msg_size)

wait_for_local_storage_truncate(self.redpanda,
topic=topic_name,
target_bytes=local_retention)

@cluster(num_nodes=3)
@matrix(cloud_storage_type=get_cloud_storage_type(
applies_only_on=[CloudStorageType.S3]))
def test_cloud_storage_sticky_enablement_v22_2_to_v22_3(
self, cloud_storage_type):
"""
In Redpanda 22.3, the cluster defaults for cloud storage change
from being applied at runtime to being sticky at creation time,
or at upgrade time.
"""
self.install_and_start()

# Switch on tiered storage using cluster properties, not topic properties
self.redpanda.set_cluster_config(
{
'cloud_storage_enable_remote_read': True,
'cloud_storage_enable_remote_write': True
},
# This shouldn't require a restart, but it does in Redpanda < 22.3
True)

topic = "test-topic"
self.rpk.create_topic(topic)
described = self.rpk.describe_topic_configs(topic)
assert described['redpanda.remote.write'] == ('true', 'DEFAULT_CONFIG')
assert described['redpanda.remote.read'] == ('true', 'DEFAULT_CONFIG')

# Upgrade to Redpanda latest
self.installer.install(self.redpanda.nodes, (22, 3))
self.redpanda.restart_nodes(self.redpanda.nodes)

# Wait for properties migration to run
self.redpanda.await_feature("cloud_retention",
active=True,
timeout_sec=30)

self.logger.info(
f"Config status after upgrade: {self.redpanda._admin.get_cluster_config_status()}"
)

# Properties should still be set, but migrated to topic-level
described = self.rpk.describe_topic_configs(topic)
assert described['redpanda.remote.write'] == ('true', 'DEFAULT_CONFIG')
assert described['redpanda.remote.read'] == ('true', 'DEFAULT_CONFIG')

# A new topic picks up these properties too
self.rpk.create_topic("created-after-enabled")
described = self.rpk.describe_topic_configs("created-after-enabled")
assert described['redpanda.remote.write'] == ('true', 'DEFAULT_CONFIG')
assert described['redpanda.remote.read'] == ('true', 'DEFAULT_CONFIG')

# Switching off cluster defaults shoudln't affect existing topic
self.redpanda.set_cluster_config(
{
'cloud_storage_enable_remote_read': False,
'cloud_storage_enable_remote_write': False
}, False)
described = self.rpk.describe_topic_configs(topic)
assert described['redpanda.remote.write'] == ('true',
'DYNAMIC_TOPIC_CONFIG')
assert described['redpanda.remote.read'] == ('true',
'DYNAMIC_TOPIC_CONFIG')

# A newly created topic should have tiered storage switched off
self.rpk.create_topic("created-after-disabled")
described = self.rpk.describe_topic_configs("created-after-disabled")
assert described['redpanda.remote.write'] == ('false',
'DEFAULT_CONFIG')
assert described['redpanda.remote.read'] == ('false', 'DEFAULT_CONFIG')

# Nothing in this test guarantees that partition manifests will be uploaded.
self.redpanda.si_settings.set_expected_damage({"ntpr_no_manifest"})

@cluster(num_nodes=3)
@matrix(cloud_storage_type=get_cloud_storage_type(
applies_only_on=[CloudStorageType.S3]))
def test_retention_config_on_upgrade_from_v22_2_to_v22_3(
self, cloud_storage_type):
self.install_and_start()

self.rpk.create_topic("test-topic-with-retention",
config={"retention.bytes": 10000})

local_retention = 10000
self.rpk.create_topic(
"test-si-topic-with-retention",
config={
"retention.bytes": str(local_retention),
"redpanda.remote.write": "true",
"redpanda.remote.read": "true",
# Small segment.bytes so that we can readily
# get data into S3 by writing small amounts.
"segment.bytes": 1000000
})

# Write a few megabytes of data, enough to spill to S3. This will be used
# later for checking that deletion behavior is correct on legacy topics.
self._populate_tiered_storage_topic("test-si-topic-with-retention",
local_retention)

# This topic is like "test-si-topic-with-retention", but instead
# of settings its properties at creation time, set them via
# alter messages: these follow a different code path in
# kafka and controller, and are serialized using different structures
# than the structures used in topic creation.
self.rpk.create_topic("test-si-topic-with-retention-altered")
self.rpk.alter_topic_config("test-si-topic-with-retention-altered",
"retention.bytes", 10000)
self.rpk.alter_topic_config("test-si-topic-with-retention-altered",
"redpanda.remote.write", "true")

# Check our alter operations applied properly
# (see https://github.com/redpanda-data/redpanda/issues/6772)
after_alter = self.rpk.describe_topic_configs(
"test-si-topic-with-retention-altered")
assert after_alter['redpanda.remote.write'][0] == 'true'
assert after_alter['retention.bytes'][0] == '10000'

# TODO: this test currently exercises 22.2 (serde) encoding to
# 22.3 (newer serde) encoding. It should be extended to also
# cover the case of creating topics in version 22.1, upgrading
# to 22.2, and then to 22.3, to check the ADL encoding of messages
# in the controller log.

self.installer.install(
self.redpanda.nodes,
(22, 3),
)

self.redpanda.restart_nodes(self.redpanda.nodes)

# Wait for any migration steps to complete
self.redpanda.await_feature("cloud_retention",
active=True,
timeout_sec=30)

non_si_configs = self.rpk.describe_topic_configs(
"test-topic-with-retention")
si_configs = self.rpk.describe_topic_configs(
"test-si-topic-with-retention")
si_altered_configs = self.rpk.describe_topic_configs(
"test-si-topic-with-retention-altered")

# "test-topic-with-retention" did not have remote write enabled
# at the time of the upgrade, so retention.* configs are preserved
# and retention.local.target.* configs are set to their default values,
# but ignored
self.logger.debug(f"Checking config {non_si_configs}")
assert non_si_configs["retention.bytes"] == ("10000",
"DYNAMIC_TOPIC_CONFIG")
assert non_si_configs["retention.ms"][1] == "DEFAULT_CONFIG"

assert non_si_configs["retention.local.target.bytes"] == (
"-1", "DEFAULT_CONFIG")
assert non_si_configs["retention.local.target.ms"][
1] == "DEFAULT_CONFIG"

# 'test-si-topic-with-retention' was enabled from remote write
# at the time of the upgrade, so retention.local.target.* configs
# should be initialised from retention.* and retention.* configs should
# be disabled.
for conf in (si_configs, si_altered_configs):
self.logger.debug(f"Checking config: {conf}")
assert conf['redpanda.remote.write'][0] == 'true'
assert conf["retention.bytes"][0] == "-1"
assert conf["retention.ms"][0] == "-1"

assert conf["retention.local.target.bytes"] == (
"10000", "DYNAMIC_TOPIC_CONFIG")
assert conf["retention.local.target.ms"][1] == "DEFAULT_CONFIG"
assert conf["redpanda.remote.delete"][0] == "false"

# After upgrade, newly created topics should have remote.delete
# enabled by default, and interpret assignments to retention properties
# literally (no mapping of retention -> retention.local)
for (new_topic_name,
enable_si) in [("test-topic-post-upgrade-nosi", False),
("test-topic-post-upgrade-si", True)]:

segment_bytes = 1000000
local_retention = segment_bytes * 2
retention_bytes = segment_bytes * 10
create_config = {
"retention.bytes": retention_bytes,
"retention.local.target.bytes": local_retention,
"segment.bytes": segment_bytes
}
if enable_si:
create_config['redpanda.remote.write'] = 'true'
create_config['redpanda.remote.read'] = 'true'

self.rpk.create_topic(new_topic_name, config=create_config)
new_config = self.rpk.describe_topic_configs(new_topic_name)
assert new_config["redpanda.remote.delete"][0] == "true"
assert new_config["retention.bytes"] == (str(retention_bytes),
"DYNAMIC_TOPIC_CONFIG")
assert new_config["retention.ms"][1] == "DEFAULT_CONFIG"
assert new_config["retention.local.target.ms"][
1] == "DEFAULT_CONFIG"
assert new_config["retention.local.target.bytes"] == (
str(local_retention), "DYNAMIC_TOPIC_CONFIG")
if enable_si:
assert new_config['redpanda.remote.write'][0] == "true"
assert new_config['redpanda.remote.read'][0] == "true"

# The remote.delete property is applied irrespective of whether
# the topic is initially tiered storage enabled.
assert new_config['redpanda.remote.delete'][0] == "true"

if enable_si:
self._populate_tiered_storage_topic(new_topic_name,
local_retention)

# A newly created tiered storage topic should have its data deleted
# in S3 when the topic is deleted
self._delete_tiered_storage_topic("test-topic-post-upgrade-si", True)

# Ensure that the `redpanda.remote.delete==false` configuration is
# really taking effect, by deleting a legacy topic and ensuring data is
# left behind in S3
self._delete_tiered_storage_topic("test-si-topic-with-retention",
False)

def _delete_tiered_storage_topic(self, topic_name: str,
expect_s3_deletion: bool):
self.logger.debug(f"Deleting {topic_name} and checking S3 result")

before_objects = set(
o.key
for o in self.cloud_storage_client.list_objects(self._s3_bucket)
if topic_name in o.key)

# Test is meaningless if there were no objects to start with
assert len(before_objects) > 0

self.rpk.delete_topic(topic_name)

def is_empty():
return sum(1 for o in self.cloud_storage_client.list_objects(
self._s3_bucket) if topic_name in o.key) == 0

if expect_s3_deletion:
wait_until(is_empty, timeout_sec=10, backoff_sec=1)
else:
# When we expect objects to remain, require that they remain for
# at least some time, to avoid false-passing if they were deleted with
# some small delay.
sleep(10)

after_objects = set(
o.key
for o in self.cloud_storage_client.list_objects(self._s3_bucket)
if topic_name in o.key)
deleted_objects = before_objects - after_objects
if expect_s3_deletion:
assert deleted_objects
else:
self.logger.debug(
f"deleted objects (should be empty): {deleted_objects}")
assert len(deleted_objects) == 0

@cluster(num_nodes=3)
@matrix(cloud_storage_type=get_cloud_storage_type(
applies_only_on=[CloudStorageType.S3]))
def test_retention_upgrade_with_cluster_remote_write(
self, cloud_storage_type):
"""
Validate how the cluster-wide cloud_storage_enable_remote_write
is handled on upgrades from <=22.2
"""
self.install_and_start()

self.redpanda.set_cluster_config(
{"cloud_storage_enable_remote_write": "true"}, expect_restart=True)

self.rpk.create_topic("test-topic-with-remote-write",
config={"retention.bytes": 10000})

self.rpk.create_topic("test-topic-without-remote-write",
config={
"retention.bytes": 10000,
"redpanda.remote.write": "false"
})

self.installer.install(
self.redpanda.nodes,
(22, 3),
)

self.redpanda.restart_nodes(self.redpanda.nodes)

# Wait for any migration steps to complete
self.redpanda.await_feature('cloud_retention',
active=True,
timeout_sec=30)

# Because legacy Redpanda treated cloud_storage_enable_remote_write as
# an override to the topic property (even if the topic remote.write was explicitly
# set to false), our two topics should end up with identical config: both
# with SI enabled and with their retention settings transposed.
for topic in [
"test-topic-with-remote-write",
"test-topic-without-remote-write"
]:
config = self.rpk.describe_topic_configs(topic)
assert config["retention.ms"] == ("-1", "DYNAMIC_TOPIC_CONFIG")
assert config["retention.bytes"] == ("-1", "DYNAMIC_TOPIC_CONFIG")
assert config["retention.local.target.bytes"] == (
"10000", "DYNAMIC_TOPIC_CONFIG")
assert config["retention.local.target.ms"][1] == "DEFAULT_CONFIG"

# The ntp manifest may not have been uploaded yet, but that's fine
# since this test focuses on the topic config change.
self.redpanda.si_settings.set_expected_damage({"ntpr_no_manifest"})

0 comments on commit 5f7ce53

Please sign in to comment.