From c3d38ea9c18f6b211c10f630bc48bc20bd2c7a4f Mon Sep 17 00:00:00 2001 From: Brian Tiemann Date: Tue, 3 Dec 2024 15:15:03 -0500 Subject: [PATCH 1/9] Wait until job1 is scheduled before enqueueing job2 --- netbox/netbox/api/pagination.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/netbox/api/pagination.py b/netbox/netbox/api/pagination.py index 5ecade2640..0a7daedaa7 100644 --- a/netbox/netbox/api/pagination.py +++ b/netbox/netbox/api/pagination.py @@ -39,7 +39,7 @@ def paginate_queryset(self, queryset, request, view=None): def get_limit(self, request): if self.limit_query_param: try: - limit = int(request.query_params[self.limit_query_param]) + limit = int(request.query_params.get(self.limit_query_param, 0)) if limit < 0: raise ValueError() # Enforce maximum page size, if defined From af2768aa3dcf59c8c6f6557df5b9327a6934f83d Mon Sep 17 00:00:00 2001 From: Brian Tiemann Date: Wed, 4 Dec 2024 12:11:26 -0500 Subject: [PATCH 2/9] Clamp limit=0 to default_limit --- netbox/netbox/api/pagination.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/netbox/api/pagination.py b/netbox/netbox/api/pagination.py index 0a7daedaa7..1270dca247 100644 --- a/netbox/netbox/api/pagination.py +++ b/netbox/netbox/api/pagination.py @@ -45,7 +45,7 @@ def get_limit(self, request): # Enforce maximum page size, if defined MAX_PAGE_SIZE = get_config().MAX_PAGE_SIZE if MAX_PAGE_SIZE: - return MAX_PAGE_SIZE if limit == 0 else min(limit, MAX_PAGE_SIZE) + return min(self.default_limit, MAX_PAGE_SIZE) if limit == 0 else min(limit, MAX_PAGE_SIZE) return limit except (KeyError, ValueError): pass From 647c7c9579b277d9338d17c37b4daab1e027d54a Mon Sep 17 00:00:00 2001 From: Brian Tiemann Date: Thu, 5 Dec 2024 15:49:36 -0500 Subject: [PATCH 3/9] Handle unspecified limit explicitly so as to return min(PAGINATE_COUNT, MAX_PAGE_SIZE) --- netbox/netbox/api/pagination.py | 4 +++- netbox/utilities/tests/test_api.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/netbox/netbox/api/pagination.py b/netbox/netbox/api/pagination.py index 1270dca247..1f4f88a0a5 100644 --- a/netbox/netbox/api/pagination.py +++ b/netbox/netbox/api/pagination.py @@ -38,12 +38,14 @@ def paginate_queryset(self, queryset, request, view=None): def get_limit(self, request): if self.limit_query_param: + MAX_PAGE_SIZE = get_config().MAX_PAGE_SIZE + if self.limit_query_param not in request.query_params: + return min(self.default_limit, MAX_PAGE_SIZE) try: limit = int(request.query_params.get(self.limit_query_param, 0)) if limit < 0: raise ValueError() # Enforce maximum page size, if defined - MAX_PAGE_SIZE = get_config().MAX_PAGE_SIZE if MAX_PAGE_SIZE: return min(self.default_limit, MAX_PAGE_SIZE) if limit == 0 else min(limit, MAX_PAGE_SIZE) return limit diff --git a/netbox/utilities/tests/test_api.py b/netbox/utilities/tests/test_api.py index ba0c3c4f82..1a62d36bb9 100644 --- a/netbox/utilities/tests/test_api.py +++ b/netbox/utilities/tests/test_api.py @@ -144,6 +144,18 @@ def test_default_page_size(self): self.assertIsNone(response.data['previous']) self.assertEqual(len(response.data['results']), page_size) + @override_settings(MAX_PAGE_SIZE=30) + def test_default_page_size_with_small_max_page_size(self): + response = self.client.get(self.url, format='json', **self.header) + page_size = get_config().MAX_PAGE_SIZE + self.assertLess(page_size, 100, "Default page size not sufficient for data set") + + self.assertHttpStatus(response, status.HTTP_200_OK) + self.assertEqual(response.data['count'], 100) + self.assertTrue(response.data['next'].endswith(f'?limit={page_size}&offset={page_size}')) + self.assertIsNone(response.data['previous']) + self.assertEqual(len(response.data['results']), page_size) + def test_custom_page_size(self): response = self.client.get(f'{self.url}?limit=10', format='json', **self.header) From 020386f8a143126aee01b5481fbedb9bbc9efc33 Mon Sep 17 00:00:00 2001 From: Brian Tiemann Date: Thu, 5 Dec 2024 16:53:42 -0500 Subject: [PATCH 4/9] Revert original min() --- netbox/netbox/api/pagination.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/netbox/api/pagination.py b/netbox/netbox/api/pagination.py index 1f4f88a0a5..f87a92ab22 100644 --- a/netbox/netbox/api/pagination.py +++ b/netbox/netbox/api/pagination.py @@ -47,7 +47,7 @@ def get_limit(self, request): raise ValueError() # Enforce maximum page size, if defined if MAX_PAGE_SIZE: - return min(self.default_limit, MAX_PAGE_SIZE) if limit == 0 else min(limit, MAX_PAGE_SIZE) + return MAX_PAGE_SIZE if limit == 0 else min(limit, MAX_PAGE_SIZE) return limit except (KeyError, ValueError): pass From acecb3a81c6784b5cb90afc2da5f6b57e0e09c99 Mon Sep 17 00:00:00 2001 From: Brian Tiemann Date: Tue, 10 Dec 2024 14:15:18 -0500 Subject: [PATCH 5/9] Coerce MAX_PAGE_SIZE to be at least PAGINATE_COUNT --- netbox/netbox/api/pagination.py | 6 +++--- netbox/utilities/tests/test_api.py | 11 ++++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/netbox/netbox/api/pagination.py b/netbox/netbox/api/pagination.py index f87a92ab22..961d524776 100644 --- a/netbox/netbox/api/pagination.py +++ b/netbox/netbox/api/pagination.py @@ -39,10 +39,10 @@ def paginate_queryset(self, queryset, request, view=None): def get_limit(self, request): if self.limit_query_param: MAX_PAGE_SIZE = get_config().MAX_PAGE_SIZE - if self.limit_query_param not in request.query_params: - return min(self.default_limit, MAX_PAGE_SIZE) + if MAX_PAGE_SIZE: + MAX_PAGE_SIZE = max(MAX_PAGE_SIZE, self.default_limit) try: - limit = int(request.query_params.get(self.limit_query_param, 0)) + limit = int(request.query_params[self.limit_query_param]) if limit < 0: raise ValueError() # Enforce maximum page size, if defined diff --git a/netbox/utilities/tests/test_api.py b/netbox/utilities/tests/test_api.py index 1a62d36bb9..2c3ba0566f 100644 --- a/netbox/utilities/tests/test_api.py +++ b/netbox/utilities/tests/test_api.py @@ -148,13 +148,14 @@ def test_default_page_size(self): def test_default_page_size_with_small_max_page_size(self): response = self.client.get(self.url, format='json', **self.header) page_size = get_config().MAX_PAGE_SIZE + paginate_count = get_config().PAGINATE_COUNT self.assertLess(page_size, 100, "Default page size not sufficient for data set") self.assertHttpStatus(response, status.HTTP_200_OK) self.assertEqual(response.data['count'], 100) - self.assertTrue(response.data['next'].endswith(f'?limit={page_size}&offset={page_size}')) + self.assertTrue(response.data['next'].endswith(f'?limit={paginate_count}&offset={paginate_count}')) self.assertIsNone(response.data['previous']) - self.assertEqual(len(response.data['results']), page_size) + self.assertEqual(len(response.data['results']), paginate_count) def test_custom_page_size(self): response = self.client.get(f'{self.url}?limit=10', format='json', **self.header) @@ -165,15 +166,15 @@ def test_custom_page_size(self): self.assertIsNone(response.data['previous']) self.assertEqual(len(response.data['results']), 10) - @override_settings(MAX_PAGE_SIZE=20) + @override_settings(MAX_PAGE_SIZE=80) def test_max_page_size(self): response = self.client.get(f'{self.url}?limit=0', format='json', **self.header) self.assertHttpStatus(response, status.HTTP_200_OK) self.assertEqual(response.data['count'], 100) - self.assertTrue(response.data['next'].endswith('?limit=20&offset=20')) + self.assertTrue(response.data['next'].endswith('?limit=80&offset=80')) self.assertIsNone(response.data['previous']) - self.assertEqual(len(response.data['results']), 20) + self.assertEqual(len(response.data['results']), 80) @override_settings(MAX_PAGE_SIZE=0) def test_max_page_size_disabled(self): From 5dd93c096d64292a034d1e1f79a5955b63f7f7f1 Mon Sep 17 00:00:00 2001 From: Brian Tiemann Date: Wed, 11 Dec 2024 16:19:26 -0500 Subject: [PATCH 6/9] Raise ImproperlyConfigured error if MAX_PAGE_SIZE < PAGINATE_COUNT --- netbox/netbox/api/pagination.py | 4 +--- netbox/netbox/settings.py | 6 ++++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/netbox/netbox/api/pagination.py b/netbox/netbox/api/pagination.py index 961d524776..5ecade2640 100644 --- a/netbox/netbox/api/pagination.py +++ b/netbox/netbox/api/pagination.py @@ -38,14 +38,12 @@ def paginate_queryset(self, queryset, request, view=None): def get_limit(self, request): if self.limit_query_param: - MAX_PAGE_SIZE = get_config().MAX_PAGE_SIZE - if MAX_PAGE_SIZE: - MAX_PAGE_SIZE = max(MAX_PAGE_SIZE, self.default_limit) try: limit = int(request.query_params[self.limit_query_param]) if limit < 0: raise ValueError() # Enforce maximum page size, if defined + MAX_PAGE_SIZE = get_config().MAX_PAGE_SIZE if MAX_PAGE_SIZE: return MAX_PAGE_SIZE if limit == 0 else min(limit, MAX_PAGE_SIZE) return limit diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index a8ac68d4dc..cad076bc33 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -129,8 +129,10 @@ LOGIN_REQUIRED = getattr(configuration, 'LOGIN_REQUIRED', True) LOGIN_TIMEOUT = getattr(configuration, 'LOGIN_TIMEOUT', None) LOGOUT_REDIRECT_URL = getattr(configuration, 'LOGOUT_REDIRECT_URL', 'home') +MAX_PAGE_SIZE = getattr(configuration, 'MAX_PAGE_SIZE', 1000) MEDIA_ROOT = getattr(configuration, 'MEDIA_ROOT', os.path.join(BASE_DIR, 'media')).rstrip('/') METRICS_ENABLED = getattr(configuration, 'METRICS_ENABLED', False) +PAGINATE_COUNT = getattr(configuration, 'PAGINATE_COUNT', 50) PLUGINS = getattr(configuration, 'PLUGINS', []) PLUGINS_CONFIG = getattr(configuration, 'PLUGINS_CONFIG', {}) QUEUE_MAPPINGS = getattr(configuration, 'QUEUE_MAPPINGS', {}) @@ -685,6 +687,10 @@ def _setting(name, default=None): 'VIEW_NAME_FUNCTION': 'utilities.api.get_view_name', } +if MAX_PAGE_SIZE < PAGINATE_COUNT: + raise ImproperlyConfigured(f'MAX_PAGE_SIZE ({MAX_PAGE_SIZE}) cannot be less than PAGINATE_COUNT ({PAGINATE_COUNT}).') + + # # DRF Spectacular # From 5087a1111a9ba4126a7ea492bfaae5f6e71957cb Mon Sep 17 00:00:00 2001 From: Brian Tiemann Date: Wed, 11 Dec 2024 16:22:53 -0500 Subject: [PATCH 7/9] Revert test behavior --- netbox/utilities/tests/test_api.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/netbox/utilities/tests/test_api.py b/netbox/utilities/tests/test_api.py index 2c3ba0566f..94be690918 100644 --- a/netbox/utilities/tests/test_api.py +++ b/netbox/utilities/tests/test_api.py @@ -144,7 +144,7 @@ def test_default_page_size(self): self.assertIsNone(response.data['previous']) self.assertEqual(len(response.data['results']), page_size) - @override_settings(MAX_PAGE_SIZE=30) + @override_settings(MAX_PAGE_SIZE=20) def test_default_page_size_with_small_max_page_size(self): response = self.client.get(self.url, format='json', **self.header) page_size = get_config().MAX_PAGE_SIZE @@ -166,15 +166,15 @@ def test_custom_page_size(self): self.assertIsNone(response.data['previous']) self.assertEqual(len(response.data['results']), 10) - @override_settings(MAX_PAGE_SIZE=80) + @override_settings(MAX_PAGE_SIZE=20) def test_max_page_size(self): response = self.client.get(f'{self.url}?limit=0', format='json', **self.header) self.assertHttpStatus(response, status.HTTP_200_OK) self.assertEqual(response.data['count'], 100) - self.assertTrue(response.data['next'].endswith('?limit=80&offset=80')) + self.assertTrue(response.data['next'].endswith('?limit=20&offset=20')) self.assertIsNone(response.data['previous']) - self.assertEqual(len(response.data['results']), 80) + self.assertEqual(len(response.data['results']), 20) @override_settings(MAX_PAGE_SIZE=0) def test_max_page_size_disabled(self): From b141f8c2d8656cdc7052f72383654a97f22ff6e2 Mon Sep 17 00:00:00 2001 From: Brian Tiemann Date: Thu, 12 Dec 2024 08:28:13 -0500 Subject: [PATCH 8/9] Revert "Revert test behavior" This reverts commit 5087a1111a9ba4126a7ea492bfaae5f6e71957cb. --- netbox/utilities/tests/test_api.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/netbox/utilities/tests/test_api.py b/netbox/utilities/tests/test_api.py index 94be690918..2c3ba0566f 100644 --- a/netbox/utilities/tests/test_api.py +++ b/netbox/utilities/tests/test_api.py @@ -144,7 +144,7 @@ def test_default_page_size(self): self.assertIsNone(response.data['previous']) self.assertEqual(len(response.data['results']), page_size) - @override_settings(MAX_PAGE_SIZE=20) + @override_settings(MAX_PAGE_SIZE=30) def test_default_page_size_with_small_max_page_size(self): response = self.client.get(self.url, format='json', **self.header) page_size = get_config().MAX_PAGE_SIZE @@ -166,15 +166,15 @@ def test_custom_page_size(self): self.assertIsNone(response.data['previous']) self.assertEqual(len(response.data['results']), 10) - @override_settings(MAX_PAGE_SIZE=20) + @override_settings(MAX_PAGE_SIZE=80) def test_max_page_size(self): response = self.client.get(f'{self.url}?limit=0', format='json', **self.header) self.assertHttpStatus(response, status.HTTP_200_OK) self.assertEqual(response.data['count'], 100) - self.assertTrue(response.data['next'].endswith('?limit=20&offset=20')) + self.assertTrue(response.data['next'].endswith('?limit=80&offset=80')) self.assertIsNone(response.data['previous']) - self.assertEqual(len(response.data['results']), 20) + self.assertEqual(len(response.data['results']), 80) @override_settings(MAX_PAGE_SIZE=0) def test_max_page_size_disabled(self): From 14d6fd1bc1d3c178acf796bde918f70953f00f11 Mon Sep 17 00:00:00 2001 From: Brian Tiemann Date: Thu, 12 Dec 2024 08:28:23 -0500 Subject: [PATCH 9/9] Revert "Raise ImproperlyConfigured error if MAX_PAGE_SIZE < PAGINATE_COUNT" This reverts commit 5dd93c096d64292a034d1e1f79a5955b63f7f7f1. --- netbox/netbox/api/pagination.py | 4 +++- netbox/netbox/settings.py | 6 ------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/netbox/netbox/api/pagination.py b/netbox/netbox/api/pagination.py index 5ecade2640..961d524776 100644 --- a/netbox/netbox/api/pagination.py +++ b/netbox/netbox/api/pagination.py @@ -38,12 +38,14 @@ def paginate_queryset(self, queryset, request, view=None): def get_limit(self, request): if self.limit_query_param: + MAX_PAGE_SIZE = get_config().MAX_PAGE_SIZE + if MAX_PAGE_SIZE: + MAX_PAGE_SIZE = max(MAX_PAGE_SIZE, self.default_limit) try: limit = int(request.query_params[self.limit_query_param]) if limit < 0: raise ValueError() # Enforce maximum page size, if defined - MAX_PAGE_SIZE = get_config().MAX_PAGE_SIZE if MAX_PAGE_SIZE: return MAX_PAGE_SIZE if limit == 0 else min(limit, MAX_PAGE_SIZE) return limit diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index cad076bc33..a8ac68d4dc 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -129,10 +129,8 @@ LOGIN_REQUIRED = getattr(configuration, 'LOGIN_REQUIRED', True) LOGIN_TIMEOUT = getattr(configuration, 'LOGIN_TIMEOUT', None) LOGOUT_REDIRECT_URL = getattr(configuration, 'LOGOUT_REDIRECT_URL', 'home') -MAX_PAGE_SIZE = getattr(configuration, 'MAX_PAGE_SIZE', 1000) MEDIA_ROOT = getattr(configuration, 'MEDIA_ROOT', os.path.join(BASE_DIR, 'media')).rstrip('/') METRICS_ENABLED = getattr(configuration, 'METRICS_ENABLED', False) -PAGINATE_COUNT = getattr(configuration, 'PAGINATE_COUNT', 50) PLUGINS = getattr(configuration, 'PLUGINS', []) PLUGINS_CONFIG = getattr(configuration, 'PLUGINS_CONFIG', {}) QUEUE_MAPPINGS = getattr(configuration, 'QUEUE_MAPPINGS', {}) @@ -687,10 +685,6 @@ def _setting(name, default=None): 'VIEW_NAME_FUNCTION': 'utilities.api.get_view_name', } -if MAX_PAGE_SIZE < PAGINATE_COUNT: - raise ImproperlyConfigured(f'MAX_PAGE_SIZE ({MAX_PAGE_SIZE}) cannot be less than PAGINATE_COUNT ({PAGINATE_COUNT}).') - - # # DRF Spectacular #