Closes netbox-community#1237: Enabled setting limit=0 to disable pagi…

…nation in API requests; added MAX_PAGE_SIZE configuration setting

docs/api/overview.md

@@ -136,3 +136,8 @@ The response will return devices 1 through 100. The URL provided in the `next` a
     "results": [...]
 }
 ```
+
+The maximum number of objects that can be returned is limited by the [`MAX_PAGE_SIZE`](../configuration/optional-settings/#max_page_size) setting, which is 1000 by default. Setting this to `0` or `None` will remove the maximum limit. An API consumer can then pass `?limit=0` to retrieve _all_ matching objects with a single request.
+
+!!! warning
+    Disabling the page size limit introduces a potential for very resource-intensive requests, since one API request can effectively retrieve an entire table from the database.

docs/configuration/optional-settings.md

@@ -99,6 +99,14 @@ Setting this to True will display a "maintenance mode" banner at the top of ever
 
 ---
 
+## MAX_PAGE_SIZE
+
+Default: 1000
+
+An API consumer can request an arbitrary number of objects by appending the "limit" parameter to the URL (e.g. `?limit=1000`). This setting defines the maximum limit. Setting it to `0` or `None` will allow an API consumer to request all objects by specifying `?limit=0`.
+
+---
+
 ## NETBOX_USERNAME
 
 ## NETBOX_PASSWORD

netbox/netbox/configuration.example.py

@@ -79,6 +79,11 @@
 # Setting this to True will display a "maintenance mode" banner at the top of every page.
 MAINTENANCE_MODE = False
 
+# An API consumer can request an arbitrary number of objects =by appending the "limit" parameter to the URL (e.g.
+# "?limit=1000"). This setting defines the maximum limit. Setting it to 0 or None will allow an API consumer to request
+# all objects by specifying "?limit=0".
+MAX_PAGE_SIZE = 1000
+
 # Credentials that NetBox will use to access live devices (future use).
 NETBOX_USERNAME = ''
 NETBOX_PASSWORD = ''

netbox/netbox/settings.py

@@ -48,6 +48,7 @@
 BANNER_BOTTOM = getattr(configuration, 'BANNER_BOTTOM', False)
 PREFER_IPV4 = getattr(configuration, 'PREFER_IPV4', False)
 ENFORCE_GLOBAL_UNIQUE = getattr(configuration, 'ENFORCE_GLOBAL_UNIQUE', False)
+MAX_PAGE_SIZE = getattr(configuration, 'MAX_PAGE_SIZE', 1000)
 CORS_ORIGIN_ALLOW_ALL = getattr(configuration, 'CORS_ORIGIN_ALLOW_ALL', False)
 CORS_ORIGIN_WHITELIST = getattr(configuration, 'CORS_ORIGIN_WHITELIST', [])
 CORS_ORIGIN_REGEX_WHITELIST = getattr(configuration, 'CORS_ORIGIN_REGEX_WHITELIST', [])
@@ -208,7 +209,7 @@
     'DEFAULT_FILTER_BACKENDS': (
         'rest_framework.filters.DjangoFilterBackend',
     ),
-    'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination',
+    'DEFAULT_PAGINATION_CLASS': 'utilities.api.OptionalLimitOffsetPagination',
     'DEFAULT_PERMISSION_CLASSES': (
         'utilities.api.TokenPermissions',
     ),

netbox/utilities/api.py

@@ -5,6 +5,7 @@
 
 from rest_framework import authentication, exceptions
 from rest_framework.exceptions import APIException
+from rest_framework.pagination import LimitOffsetPagination
 from rest_framework.permissions import DjangoModelPermissions, SAFE_METHODS
 from rest_framework.serializers import Field, ValidationError
 
@@ -105,3 +106,51 @@ def get_serializer_class(self):
         if self.action in WRITE_OPERATIONS and hasattr(self, 'write_serializer_class'):
             return self.write_serializer_class
         return self.serializer_class
+
+
+class OptionalLimitOffsetPagination(LimitOffsetPagination):
+    """
+    Override the stock paginator to allow setting limit=0 to disable pagination for a request. This returns all objects
+    matching a query, but retains the same format as a paginated request. The limit can only be disabled if
+    MAX_PAGE_SIZE has been set to 0 or None.
+    """
+
+    def paginate_queryset(self, queryset, request, view=None):
+
+        try:
+            self.count = queryset.count()
+        except (AttributeError, TypeError):
+            self.count = len(queryset)
+        self.limit = self.get_limit(request)
+        self.offset = self.get_offset(request)
+        self.request = request
+
+        if self.limit and self.count > self.limit and self.template is not None:
+            self.display_page_controls = True
+
+        if self.count == 0 or self.offset > self.count:
+            return list()
+
+        if self.limit:
+            return list(queryset[self.offset:self.offset + self.limit])
+        else:
+            return list(queryset[self.offset:])
+
+    def get_limit(self, request):
+
+        if self.limit_query_param:
+            try:
+                limit = int(request.query_params[self.limit_query_param])
+                if limit < 0:
+                    raise ValueError()
+                # Enforce maximum page size, if defined
+                if settings.MAX_PAGE_SIZE:
+                    if limit == 0:
+                        return settings.MAX_PAGE_SIZE
+                    else:
+                        return min(limit, settings.MAX_PAGE_SIZE)
+                return limit
+            except (KeyError, ValueError):
+                pass
+
+        return self.default_limit