Merge pull request WeblateOrg#5 from superDross/request-retry-mechanism

Request retry mechanism

README.rst

@@ -66,16 +66,24 @@ Command line usage:
     wlc download
     wlc upload
 
-Configuration is stored in ``~/.config/weblate``:
+Configuration is stored in ``~/.config/weblate``. The key/values (``retries``,
+``method_whitelist``, ``backoff_factor``, ``status_forcelist``) are closely
+coupled with the `urllib3 parameters`_ and allows the user to configure request
+retries.
 
 .. code-block:: ini
 
     [weblate]
     url = https://hosted.weblate.org/api/
+    retries = 3
+    method_whitelist = PUT,POST,GET
+    backoff_factor = 0.2
+    status_forcelist = 429,500,502,503,504
 
     [keys]
     https://hosted.weblate.org/api/ = APIKEY
 
 .. _Weblate's REST API: https://docs.weblate.org/en/latest/api.html
 .. _Weblate documentation: https://docs.weblate.org/en/latest/wlc.html
 .. _Weblate: https://weblate.org/
+.. _urllib3 parameters: https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html#module-urllib3.util.retry

wlc/__init__.py

@@ -23,6 +23,8 @@
 
 import dateutil.parser
 import requests
+from requests.adapters import HTTPAdapter
+from requests.packages.urllib3.util.retry import Retry
 
 __version__ = "1.6"
 
@@ -55,9 +57,7 @@ def __init__(self):
 
 class IsNotMonolingual(WeblateException):
     def __init__(self):
-        super().__init__(
-            "Source strings can only be added to monolingual components"
-        )
+        super().__init__("Source strings can only be added to monolingual components")
 
 class ContentExists(WeblateException):
     def __init__(self):
@@ -69,13 +69,43 @@ def __init__(self):
 class Weblate:
     """Weblate API wrapper object."""
 
-    def __init__(self, key="", url=API_URL, config=None):
-        """Create the object, storing key and API url."""
+    def __init__(
+        self,
+        key="",
+        url=API_URL,
+        config=None,
+        retries=0,
+        status_forcelist=None,
+        method_whitelist=None,
+        backoff_factor=0,
+    ):
+        """Create the object, storing key, API url and requests retry args."""
         if config is not None:
             self.url, self.key = config.get_url_key()
+            (
+                self.retries,
+                self.status_forcelist,
+                self.method_whitelist,
+                self.backoff_factor,
+            ) = config.get_retry_options()
         else:
             self.key = key
             self.url = url
+            self.retries = retries
+            self.status_forcelist = status_forcelist
+            if method_whitelist is None:
+                self.method_whitelist = [
+                    "HEAD",
+                    "GET",
+                    "PUT",
+                    "DELETE",
+                    "OPTIONS",
+                    "TRACE",
+                ]
+            else:
+                self.method_whitelist = method_whitelist
+            self.backoff_factor = backoff_factor
+
         if not self.url.endswith("/"):
             self.url += "/"
 
@@ -138,7 +168,16 @@ def invoke_request(self, method, path, params=None, files=None):
             # JSON params to handle complex structures
             kwargs = {"json": params}
         try:
-            response = requests.request(
+            req = requests.Session()
+            retries = Retry(
+                total=self.retries,
+                backoff_factor=self.backoff_factor,
+                status_forcelist=self.status_forcelist,
+                method_whitelist=self.method_whitelist,
+            )
+            for protocol in ["http", "https"]:
+                req.mount(f"{protocol}://", HTTPAdapter(max_retries=retries))
+            response = req.request(
                 method, path, headers=headers, verify=verify_ssl, files=files, **kwargs,
             )
             response.raise_for_status()
@@ -153,8 +192,7 @@ def post(self, path, **kwargs):
 
     def _post_factory(self, prefix, path, kwargs):
         """Wrapper for posting objects."""
-        resp = self.post("/".join((prefix, path, "")), **kwargs)
-        return resp
+        return self.post("/".join((prefix, path, "")), **kwargs)
 
     def get(self, path):
         """Perform GET request on the API."""
@@ -221,14 +259,14 @@ def list_languages(self):
         return self.list_factory("languages/", Language)
 
     def _is_component_monolingual(self, path):
-        """ Determines if a component is configured monolinugally"""
+        """Determines if a component is configured monolinugally."""
         comp = self.get_component(path)
         if comp["template"]:
             return True
         return False
 
     def add_source_string(self, project, component, msgid, msgstr):
-        """Adds a source string to a monolingual base file"""
+        """Adds a source string to a monolingual base file."""
         source_language = self.get_project(project)["source_language"]["code"]
         is_monolingual = self._is_component_monolingual(f"{project}/{component}")
         if not is_monolingual:
@@ -559,7 +597,7 @@ def delete(self):
         self.weblate.raw_request("delete", self._url)
 
     def add_source_string(self, msgid, msgstr):
-        """Adds a source string to a monolingual base file"""
+        """Adds a source string to a monolingual base file."""
         return self.weblate.add_source_string(
             project=self.project.slug, component=self.slug, msgid=msgid, msgstr=msgstr
         )

wlc/config.py

@@ -43,6 +43,12 @@ def set_defaults(self):
         self.add_section(self.section)
         self.set(self.section, "key", "")
         self.set(self.section, "url", wlc.API_URL)
+        self.set(self.section, "retries", 0)
+        self.set(self.section, "status_forcelist", None)
+        self.set(
+            self.section, "method_whitelist", "HEAD\nTRACE\nDELETE\nOPTIONS\nPUT\nGET"
+        )
+        self.set(self.section, "backoff_factor", 0)
 
     def load(self, path=None):
         """Load configuration from XDG paths."""
@@ -71,3 +77,12 @@ def get_url_key(self):
             except NoOptionError:
                 key = ""
         return url, key
+
+    def get_retry_options(self):
+        retries = int(self.get(self.section, "retries"))
+        status_forcelist = self.get(self.section, "status_forcelist")
+        if status_forcelist is not None:
+            status_forcelist = [int(option) for option in status_forcelist.split(",")]
+        method_whitelist = self.get(self.section, "method_whitelist").split(",")
+        backoff_factor = float(self.get(self.section, "backoff_factor"))
+        return retries, status_forcelist, method_whitelist, backoff_factor

wlc/test_data/wlc

@@ -1,5 +1,9 @@
 [weblate]
 url = https://example.net/
+retries = 999
+method_whitelist = PUT,POST
+backoff_factor = 0.2
+status_forcelist = 429,500,502,503,504
 
 [withkey]
 url = http://127.0.0.1:8000/api/

wlc/test_main.py

@@ -118,13 +118,47 @@ def test_config_cwd(self):
         finally:
             os.chdir(current)
 
+    def test_default_config_values(self):
+        """Test default parser values."""
+        config = WeblateConfig()
+        self.assertEqual(config.get("weblate", "key"), "")
+        self.assertEqual(config.get("weblate", "retries"), 0)
+        self.assertEqual(
+            config.get("weblate", "method_whitelist"),
+            "HEAD\nTRACE\nDELETE\nOPTIONS\nPUT\nGET",
+        )
+        self.assertEqual(config.get("weblate", "backoff_factor"), 0)
+        self.assertEqual(config.get("weblate", "status_forcelist"), None)
+
     def test_parsing(self):
         """Test config file parsing."""
         config = WeblateConfig()
         self.assertEqual(config.get("weblate", "url"), wlc.API_URL)
         config.load()
         config.load(TEST_CONFIG)
         self.assertEqual(config.get("weblate", "url"), "https://example.net/")
+        self.assertEqual(config.get("weblate", "retries"), "999")
+        self.assertEqual(config.get("weblate", "method_whitelist"), "PUT,POST")
+        self.assertEqual(config.get("weblate", "backoff_factor"), "0.2")
+        self.assertEqual(
+            config.get("weblate", "status_forcelist"), "429,500,502,503,504"
+        )
+
+    def test_get_retry_options(self):
+        """Test the get_retry_options method when all options are in config."""
+        config = WeblateConfig()
+        config.load()
+        config.load(TEST_CONFIG)
+        (
+            retries,
+            status_forcelist,
+            method_whitelist,
+            backoff_factor,
+        ) = config.get_retry_options()
+        self.assertEqual(retries, 999)
+        self.assertEqual(status_forcelist, [429, 500, 502, 503, 504])
+        self.assertEqual(method_whitelist, ["PUT", "POST"])
+        self.assertEqual(backoff_factor, 0.2)
 
     def test_argv(self):
         """Test sys.argv processing."""