Merge branch 'develop'

README.md

@@ -64,3 +64,17 @@ After installing requirements (via pip) within your environment or virtualenv:
 * `python manage.py reload_imports`
 
 At this point, the basic country and language datasets will be populated but without many optional fields or extra data.
+
+### Updating the `/exports/langnames.json` and `/exports/langnames_short.json` endpoints
+
+When languages are added or updated, run this command to update the data locally:
+
+```bash
+python manage.py rebuild_langnames
+```
+
+Switch to the master branch and run this command to update the data on the server:
+
+```bash
+ec run web python manage.py rebuild_langnames
+```

readme-dev.md

@@ -7,7 +7,7 @@ Also, the instructions assume you are using PyCharm or IntelliJ IDEA.
 
 ### Ubuntu prerequisites
 
-    sudo apt-get install libtiff5-dev libjpeg8-dev zlib1g-dev libfreetype6-dev liblcms2-dev libwebp-dev tcl8.6-dev tk8.6-dev python-tk
+    sudo apt-get install libtiff5-dev libjpeg8-dev zlib1g-dev libfreetype6-dev liblcms2-dev libwebp-dev tcl8.6-dev tk8.6-dev python-tk libpq-dev
 
 
 ### Install python versions
@@ -91,7 +91,8 @@ Now, restart the service: `sudo service postgresql restart`.
 
 Run this script to initialize the database with a dump from the production server:
 
-    ec run db --instance=primary -- pg_dump --no-owner --no-acl | ./manage.py dbshell
+    curl $(ec postgres dump) > db.dump
+    pg_restore --no-owner --no-acl --verbose -d td db.dump
 
 If you get timeout errors running the above command, this is an alternative that has worked:
 
@@ -120,6 +121,14 @@ If you get timeout errors running the above command, this is an alternative that
   * PostgreSQL repository: `deb http://apt.postgresql.org/pub/repos/apt/ trusty-pgdg main`
 
 
+### Deploying
+
+1. Switch to the develop branch (for td-demo) or the master branch (for td).
+2. Make sure changes have been pushed to Github.
+3. Run this command:
+```ec deploy```
+
+
 ### Upgrade PostgreSQL 9.3 to 9.5 on Ubuntu 14.04
 
 This was required because the `publishing_publishrequest` contains a `jsonb` field, a new type that was introduced in PostgreSQL 9.4.
@@ -144,4 +153,3 @@ wget -q https://www.postgresql.org/media/keys/ACCC4CF8.asc -O - | sudo apt-key a
 sudo apt-get update
 sudo apt-get install postgresql postgresql-contrib
 ```
-

requirements.txt

@@ -27,7 +27,7 @@ Pillow==3.0.0
 pinax-invitations==4.0.4
 pinax-theme-bootstrap==7.2.0
 pinax-types==0.10.0
-psycopg2==2.6.1
+psycopg2==2.7.4
 python-dateutil==2.4.2
 pytz==2015.7
 redis==2.10.5

td/api/urls.py

@@ -1,11 +1,18 @@
 from django.conf.urls import url
 
-from .views import QuestionnaireView, templanguages_json, lang_assignment_json, lang_assignment_changed_json
+from .views import (
+    celerybeat_healthz,
+    lang_assignment_changed_json,
+    lang_assignment_json,
+    QuestionnaireView,
+    templanguages_json
+)
 
 
 urlpatterns = [
     url(r"^questionnaire/$", QuestionnaireView.as_view(), name="questionnaire"),
     url(r"^templanguages/$", templanguages_json, name="templanguages"),
     url(r"^templanguages/assignment/$", lang_assignment_json, name="templanguages_assignment"),
     url(r"^templanguages/assignment/changed/$", lang_assignment_changed_json, name="templanguages_changed"),
+    url(r"^celerybeat/healthz/$", celerybeat_healthz),
 ]

td/api/views.py

@@ -1,10 +1,13 @@
 import json
 import logging
 
+from django.conf import settings
 from django.http import JsonResponse
 from django.views.decorators.csrf import csrf_exempt
 from django.views.generic import View
 
+from djcelery.models import PeriodicTask
+
 from td.models import TempLanguage, Country
 from td.resources.models import Questionnaire
 
@@ -107,3 +110,35 @@ def lang_assignment_json(request):
 
 def lang_assignment_changed_json(request):
     return JsonResponse(TempLanguage.lang_assigned_changed_map(), safe=False)
+
+
+def celerybeat_healthz(request):
+    if request.GET.get("key") != settings.CELERYBEAT_HEALTHZ_AUTH_KEY:
+        return JsonResponse({"message": "Not authorized."}, status=403)
+
+    succesful_tasks = []
+    failing_tasks = []
+
+    past_sixty_seconds = -60
+    for task in PeriodicTask.objects.filter(enabled=True):
+        # retrieve the estimated number of seconds until the next time the task should be ran
+        seconds_until_next_execution = task.schedule.remaining_estimate(task.last_run_at).total_seconds()
+
+        if seconds_until_next_execution > past_sixty_seconds:
+            succesful_tasks.append(task.name)
+        else:
+            # the task should have been scheduled already
+            failing_tasks.append(task.name)
+
+    healthy = True
+    status_code = 200
+    if failing_tasks:
+        healthy = False
+        status_code = 503
+
+    data = {
+        "healthy": healthy,
+        "failing": failing_tasks,
+        "succesful": succesful_tasks,
+    }
+    return JsonResponse(data, status=status_code)

td/imports/management/commands/rebuild_langnames.py

@@ -0,0 +1,10 @@
+from django.core.management.base import BaseCommand
+
+from td.tasks import update_langnames_data
+
+
+class Command(BaseCommand):
+    help = "rebuild langnames and langnames_short in the table td_jsondata"
+
+    def handle(self, *args, **options):
+        update_langnames_data()

td/models.py

@@ -438,7 +438,7 @@ def names_text(cls):
     def names_data(cls, short=False):
         languages = cls.objects.all().order_by("code")
         if short:
-            data = [dict(pk=x.pk, lc=x.lc, ln=x.ln, ang=x.ang, lr=x.lr) for x in languages]
+            data = [dict(pk=x.pk, lc=x.lc, ln=x.ln, ang=x.ang, lr=x.lr, hc=x.cc) for x in languages]
         else:
             # Filter out languages that have pending or rejected temporary language
             # NOTE: Can this be simplified or turned into a list comprehension?
@@ -450,7 +450,7 @@ def names_data(cls, short=False):
                     x = lang
                 if x:
                     data.append(dict(pk=x.pk, lc=x.lc, ln=x.ln, ang=x.ang, alt=x.alt_name_all, cc=x.cc_all, lr=x.lr,
-                                gw=x.gateway_flag, ld=x.get_direction_display()))
+                                gw=x.gateway_flag, ld=x.get_direction_display(), hc=x.cc))
         return data
 
     @classmethod

td/settings.py

@@ -256,6 +256,7 @@
 CELERY_TASK_SERIALIZER = "json"
 CELERY_ACCEPT_CONTENT = ['json', 'msgpack', 'yaml']   # for security reasons, don't allow pickle
 CELERY_RESULT_BACKEND = "redis://"
+CELERYBEAT_HEALTHZ_AUTH_KEY = os.environ.get("CELERYBEAT_HEALTHZ_AUTH_KEY", "celerybeat-healthz")
 
 REST_FRAMEWORK = {
     # Use Django's standard `django.contrib.auth` permissions,

td/tasks.py

@@ -77,6 +77,10 @@ def update_langnames_data():
     langnames.data = Language.names_data()
     langnames.save()
 
+    shorter, created = JSONData.objects.get_or_create(name="langnames_short")
+    shorter.data = Language.names_data(True)
+    shorter.save()
+
 
 @task()
 def reset_langnames_cache(short=False):

td/tests/test_models.py

@@ -319,9 +319,11 @@ def test_names_data_short(self):
         self.assertIn("ln", result[0])
         self.assertIn("ang", result[0])
         self.assertIn("lr", result[0])
+        self.assertIn("hc", result[0])
         self.assertNotIn("alt", result[0])
         self.assertNotIn("ld", result[0])
         self.assertNotIn("gw", result[0])
+        self.assertNotIn("cc", result[0])
         self.assertEqual(result[0].get("lc"), "tl")
         self.assertEqual(result[0].get("ln"), "Test Language")
 
@@ -331,6 +333,12 @@ def test_names_data(self):
         """
         result = Language.names_data()
         self.assertEqual(len(result), 1)
+        self.assertIn("pk", result[0])
+        self.assertIn("lc", result[0])
+        self.assertIn("ln", result[0])
+        self.assertIn("ang", result[0])
+        self.assertIn("lr", result[0])
+        self.assertIn("hc", result[0])
         self.assertIn("alt", result[0])
         self.assertIn("ld", result[0])
         self.assertIn("gw", result[0])

td/tests/test_tasks.py

@@ -97,6 +97,7 @@ def test_update_langnames_data(self):
         self.assertIn("lr", json)
         self.assertIn("gw", json)
         self.assertIn("ld", json)
+        self.assertIn("hc", json)
 
 
 class PostToExtAppTestCase(TestCase):

td/tests/test_views.py

@@ -20,7 +20,7 @@
 from ..resources.models import Questionnaire
 from ..views import TempLanguageListView, TempLanguageDetailView, TempLanguageUpdateView, AjaxTemporaryCode,\
     TempLanguageAdminView, TempLanguageWizardView, LanguageDetailView, codes_text_export, names_text_export,\
-    names_json_export, gateway_languages_autocomplete
+    names_json_export, names_json_export_short, gateway_languages_autocomplete
 from ..forms import TempLanguageForm
 from ..tests.models import NoSignalTestCase
 
@@ -86,6 +86,20 @@ def test_return_json(self, mock_jsondata_get):
         self.assertIsInstance(response, JsonResponse)
 
 
+class NamesJsonExportShortTestCase(TestCase):
+
+    @patch("td.models.JSONData.objects.get")
+    def test_return_json(self, mock_jsondata_get):
+        mock_jsondata_get.return_value.data = [dict(pk=1, lc="en", ln="English", ang="English", lr="Europe", hc="GB")]
+        response = names_json_export_short(None)
+
+        # Must call JSONData.objects.get(name="langnames_short")
+        mock_jsondata_get.assert_called_once_with(name="langnames_short")
+
+        # Must return JsonResponse
+        self.assertIsInstance(response, JsonResponse)
+
+
 class LanguageDetailViewTestCase(TestCase):
     def setUp(self):
         self.request = RequestFactory().get("uw/languages/")

td/urls.py

@@ -23,6 +23,7 @@
     codes_text_export,
     names_text_export,
     names_json_export,
+    names_json_export_short,
     export_svg,
     languages_autocomplete,
     gateway_languages_autocomplete,
@@ -37,6 +38,7 @@
     url(r"^exports/codes-d43.txt$", codes_text_export, name="codes_text_export"),
     url(r"^exports/langnames.txt$", names_text_export, name="names_text_export"),
     url(r"^exports/langnames.json$", names_json_export, name="names_json_export"),
+    url(r"^exports/langnames_short.json$", names_json_export_short, name="names_json_export_short"),
     url(r"^exports/gatewaylanguages-map/$", export_svg, name="gateway_languages_map_export"),
 
     url(r"^uw/", include("td.resources.urls")),

td/views.py

@@ -51,14 +51,23 @@ def names_json_export(request):
     # NOTE: Temp solution to langnames.json caching problem
     # NOTE: This is the caching way
     # data = get_langnames()
-    # NOTE: This is the direct, snychronous way
+    # NOTE: This is the direct, synchronous way
     # data = Language.names_data()
     # NOTE: This is the DB/management command way
     langnames = JSONData.objects.get(name="langnames")
     # Set safe to False to allow list instead of dict to be returned
     return JsonResponse(langnames.data, safe=False)
 
 
+def names_json_export_short(request):
+
+    # NOTE: This is the DB/management command way
+    langnames = JSONData.objects.get(name="langnames_short")
+
+    # Set safe to False to allow list instead of dict to be returned
+    return JsonResponse(langnames.data, safe=False)
+
+
 @csrf_exempt
 def export_svg(request):
     svg = request.POST.get("data")