Retries for notifications

promgen/locale/ja/LC_MESSAGES/django.po

@@ -25,6 +25,10 @@ msgstr "Expire"
 msgid "Test"
 msgstr "テスト"
 
+#: templates/promgen/sender_row.html:45
+msgid "Testing"
+msgstr "テスト中"
+
 #: templates/promgen/notifier_block.html:19
 msgid "Delete notification?"
 msgstr "Notificiationを削除しますか？"

promgen/notification/linenotify.py

@@ -2,6 +2,7 @@
 # These sources are released under the terms of the MIT license: see LICENSE
 
 import logging
+from http import HTTPStatus
 
 from django import forms
 
@@ -45,4 +46,13 @@ def _send(self, token, data):
 
         headers = {"Authorization": "Bearer %s" % token}
 
-        util.post(url, data=params, headers=headers).raise_for_status()
+        util.post_with_retry(
+            url,
+            data=params,
+            headers=headers,
+            retry_codes=(
+                HTTPStatus.GATEWAY_TIMEOUT,
+                HTTPStatus.TOO_MANY_REQUESTS,
+                HTTPStatus.SERVICE_UNAVAILABLE,
+            ),
+        ).raise_for_status()

promgen/notification/slack.py

@@ -1,6 +1,7 @@
 # These sources are released under the terms of the MIT license: see LICENSE
 
 import logging
+from http import HTTPStatus
 
 from django import forms
 
@@ -55,4 +56,13 @@ def _send(self, url, data):
             "text": message,
         }
 
-        util.post(url, json=json, **kwargs).raise_for_status()
+        util.post_with_retry(
+            url,
+            json=json,
+            retry_codes=(
+                HTTPStatus.GATEWAY_TIMEOUT,
+                HTTPStatus.TOO_MANY_REQUESTS,
+                HTTPStatus.SERVICE_UNAVAILABLE,
+            ),
+            **kwargs,
+        ).raise_for_status()

promgen/notification/webhook.py

@@ -8,6 +8,7 @@
 """
 
 import logging
+from http import HTTPStatus
 
 from django import forms
 
@@ -36,4 +37,12 @@ class NotificationWebhook(NotificationBase):
     form = FormWebhook
 
     def _send(self, url, data):
-        util.post(url, json=data).raise_for_status()
+        util.post_with_retry(
+            url,
+            json=data,
+            retry_codes=(
+                HTTPStatus.GATEWAY_TIMEOUT,
+                HTTPStatus.TOO_MANY_REQUESTS,
+                HTTPStatus.SERVICE_UNAVAILABLE,
+            ),
+        ).raise_for_status()

promgen/templates/promgen/sender_row.html

@@ -40,7 +40,12 @@
         <form method="post" action="{% url 'notifier-test' notifier.id %}" style="display: inline">
             {% csrf_token %}
             <input name="next" type="hidden" value="{{ request.get_full_path }}" />
-            <button class="btn btn-info btn-xs">{% trans "Test" %}</button>
+            <button
+                class="btn btn-info btn-xs"
+                onclick="this.disabled=true; this.innerHTML='{% trans "Testing" %}...';"
+            >
+                {% trans "Test" %}
+            </button>
         </form>
         <form method="post" action="{% url 'notifier-delete' notifier.id %}" onsubmit="return confirm('{% trans "Delete notification?" %}')" style="display: inline">
             {% csrf_token %}

promgen/util.py

@@ -2,6 +2,8 @@
 # These sources are released under the terms of the MIT license: see LICENSE
 
 import argparse
+import time
+from http import HTTPStatus
 from urllib.parse import urlsplit
 
 import requests
@@ -27,6 +29,33 @@ def post(url, data=None, json=None, **kwargs):
     return requests.post(url, data=data, json=json, **kwargs)
 
 
+def post_with_retry(
+    url,
+    data=None,
+    json=None,
+    retries: int = 3,
+    retry_codes: tuple = (HTTPStatus.TOO_MANY_REQUESTS,),
+    wait_secs: int = 3,
+    **kwargs
+):
+    """POST request with retry capability.
+
+    Thin wrapper for our util::post function that implements a retry mechanism for certain status
+    codes.
+
+    :param retries: Number of retries
+    :param retry_codes: Response status codes that will cause a retry
+    :param wait_secs: Number of seconds to wait between each retry
+    """
+    for n in range(retries):
+        resp = post(url, data=data, json=json, **kwargs)
+
+        if (resp.status_code in retry_codes) and (n < retries - 1):
+            time.sleep(wait_secs)
+        else:
+            return resp
+
+
 def get(url, params=None, **kwargs):
     headers = kwargs.setdefault("headers", {})
     headers["User-Agent"] = USER_AGENT