Merge pull request #778 from readthedocs/davidfischer/limit-classifier

Limit classifier to 20k
readthedocs · Aug 14, 2023 · a1bb723 · a1bb723
2 parents eec0349 + d1f5ce1
commit a1bb723
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/adserver/analyzer/backends/eatopics.py b/adserver/analyzer/backends/eatopics.py
@@ -25,6 +25,10 @@ class EthicalAdsTopicsBackend(TextacyAnalyzerBackend):
     # Threshold on the model
     MODEL_THRESHOLD = 0.4
 
+    # The model can quickly consume all the memory if the input is too long
+    # 20k characters is a lot of input to classify a page
+    MAX_INPUT_LENGTH = 20_000
+
     def skip_classification(self, text):
         """Return True if classification should be skipped."""
         if len(text) < self.MIN_TEXT_LENGTH: