Merge pull request #168 from DEKHTIARJonathan/dev

Merge Django 2.0 into master

.cfignore

@@ -81,58 +81,57 @@ ENV/
 # Spyder project settings
 .spyderproject
 
-# DB Files
-*/staticfiles/
+# StaticFiles
+application/staticfiles/
 
 # App Specific Files
 application/staticfiles/
 cover/
 test.py
 dummy_data.txt
 
-#CodeDeploy
-codedeploy/certificate.crt
-codedeploy/data.zip
-codedeploy/data_.zip
-codedeploy/deploy.sh
-codedeploy/feedcrunch.xyz.csr
-codedeploy/https.key
-https/deploy.sh
-fieldkeys/*
-!fieldkeys/__init__.py
+# Fixtures
+application/fixtures/feedcrunch_dump8.json
+application/fixtures/feedcrunch_dump9.json
 application/fixtures/dataradar.json
+
+# SQLite DB
 *.sqlite3
+
+# Media Files
+media/
 !media/images/user_photos/__init__.py
 !media/images/user_photos/dummy_user.png
-media/images/user_photos/
-codedeploy/Encrypted/*.zip
-codedeploy/Encrypted/*.txt
-*.model
-FeedCrunch.IO.zip
-boto-develop/build/
-application/fixtures/feedcrunch_dump8.json
-application/fixtures/feedcrunch_dump9.json
-media/images/interest_photos/
 
-# CloudFoundry Remove
+# Templates
 _unused_/
-codedeploy/
 example_files/
+
+# Keys and Certificates
 certificate/
-fieldkeys/
-lib_bin/
-media/
-scripts/
-venv/
+.crt
+.key
+
+# CI Chain
 .codeclimate.yml
-.coveragerc
-.env
-.env.dist
-.gitignore
-circle.yml
 .travis.yml
-app.json
-appspec.yml
+.pyup.yml
+*.yml
+
+# Python Testing
 pytest.ini
+.coverage
+.coveragerc
+
+# Git Files
 README.md
-*.yml
+CODE_OF_CONDUCT.md
+CONTRIBUTING.md
+LICENSE.md
+.env.dist
+.gitignore
+.github/
+.git/
+
+# Binary Folders
+lib_bin/

.coveragerc

@@ -2,4 +2,10 @@
 omit =
 	*/migrations/*
 	venv/*
-	/home/travis/virtualenv/*/lib/*/site-packages/*
+	/home/travis/virtualenv/*/lib/*/site-packages/*	
+	python3.4.6/*
+	python3.6.3/*
+	staticfiles/*
+	tests/*
+	lib/*
+	django_celery_monitor/*

.env.dist

@@ -1,5 +1,8 @@
 DATABASE_URL='postgres://user:password@server:port/dbname'
-RABBITMQ_URL='amqp://username:password@server:port/instance_name'
+RABBITMQ_URL='amqps://username:password@server:port/instance_name'
+REDIS_URL='redis://username:password@server:port'
+USE_RABBITMQ=True
+
 SECRET_KEY='################################'
 DEBUG=True
 AWS_USER='################################'

.gitignore

@@ -76,49 +76,56 @@ celerybeat-schedule
 
 # virtualenv
 venv/
+venv_linux/
 ENV/
 
 # Spyder project settings
 .spyderproject
 
-# DB Files
-*/staticfiles/
+# StaticFiles
+application/staticfiles/
 
 # App Specific Files
 application/staticfiles/
 cover/
 test.py
 dummy_data.txt
 
-#CodeDeploy
-codedeploy/certificate.crt
-codedeploy/data.zip
-codedeploy/data_.zip
-codedeploy/deploy.sh
-codedeploy/feedcrunch.xyz.csr
-codedeploy/https.key
-https/deploy.sh
-fieldkeys/*
-!fieldkeys/__init__.py
-application/fixtures/dataradar.json
+# Fixtures
+application/fixtures/*
+!application/fixtures/.gitkeep
+
+# SQLite DB
 *.sqlite3
-!media/images/user_photos/__init__.py
+
+# Media Files
+!media/
+media/*
+!media/*/
+media/*/*
+!media/*/*/
+media/*/*/*
+!media/images/user_photos/.gitkeep
 !media/images/user_photos/dummy_user.png
-media/images/user_photos/
-codedeploy/Encrypted/*.zip
-codedeploy/Encrypted/*.txt
-*.model
-FeedCrunch.IO.zip
-boto-develop/build/
-application/fixtures/feedcrunch_dump8.json
-application/fixtures/feedcrunch_dump9.json
-media/images/interest_photos/
-scripts/django_q.sql
-/flower*
-/celery*
-Procfile
-venv_linux/
-.idea/
-.key
+!media/images/interest_photos/.gitkeep
+!media/images/interest_photos/dummy_stock.jpg
+!media/estimators/.gitkeep
+
+# Templates
+_unused_/
+example_files/
+
+# Keys and Certificates
+certificate/
 .crt
-coverage.xml
+.key
+
+# Python Testing
+.coverage
+
+# PyCharm
+.idea
+celeryev.pid
+celerybeat.pid
+fieldkeys/
+.crt*

.travis.yml

@@ -12,9 +12,11 @@ python:
 
 services:
 - rabbitmq
+- redis-server
 - postgresql
 
 before_install:
+- python -c "import fcntl; fcntl.fcntl(1, fcntl.F_SETFL, 0)"
 - export DJANGO_SETTINGS_MODULE=application.settings
 - export PYTHONPATH=$HOME/builds/DEKHTIARJonathan/FeedCrunch.IO
 - virtualenv venv
@@ -51,6 +53,7 @@ env:
   global:
 
   - RABBITMQ_URL='amqp://guest:guest@localhost:5672/'
+  - REDIS_URL='redis://localhost:6379'
   - DATABASE_URL='postgres://postgres:@localhost:5432/travisci'
   - DEBUG='True'
   - EMAIL_DEFAULT_SENDER='local@local.host'
@@ -60,13 +63,14 @@ env:
   - BLUEMIX_API_GATEWAY='https://api.eu-gb.bluemix.net'
   - BLUEMIX_API_ORGANISATION='FeedCrunch'
   - BLUEMIX_API_SPACE='prod'
-
+  - USE_RABBITMQ='False'
+
   ### == CODECOV_TOKEN === ###
   - secure: "sRuP0d5h9pZRVHMo+8TjOdx80rJEb9GiR26uEM2dZZvm3oh+jfHbiqSecaLa3yo6qgyLwz3doSZx1eu7ZW6RDujEHHSg/SdHkosLzo3qKE1Mr5XPJXLtwjXcvSVYhiMVS5E/IcrOKqEi5p694GsykLwz5jVb6orCh/O8UcHHb2NP99JhKWcxFRjEpPoitNmTcigd1VJjqf7MiMq8+GqEhe4vug9XCnyUgpUM7okOkbem2DPaABwEJfZC9f0HVlJa1X1r1mGAFJuPqJpJ0sOiimZo9k0yfBSGjTCWmr+aga1ASjMJUKBCiEk+yEAaLwu6w0Zr+nYuIhLeLWDxVwJpmUBs0QFM4RX4Q+Pt3hUs9ynCWrXfcR/1hNdQD76705Bs9/4TxBwT7sOMfdVsywcpWtUPtfFTYmQL1sBMnz1GOg4ueZQHL7+GaARaZs+5XuWiaNIP0/xmFE8C+EHm2OqesgwYD/rdJxYG5VMyfyrjsfEJrZLx0TV8e9xbhEx3x9ZeTJrBV24/qDyYT7Ekn2kY+t8nA7V8NSKkleMiT+5D81PyNLJ0+YJjw/Ot+DIT0U6IkuSi5Y0eQMGq+mRfLGGY3YNCSeo1MTxhy5EO3pX4mwVJZt2QwOQwzqoJyGM8foNu6NH1hBjX6IH3B0smpmLoPoY3TCEffZOYjpSoGOqBQsE="
-  
+
   ### === CODACY_PROJECT_TOKEN === ###
   - secure: "MJ6PAxlkD+Vch1QcAyJB/isYfzqs/fgVYp5dM/Rq2pj/sisl3TtpkdNyiddlFMONH+BkxLxUnBng4PaJewrnWc6ojUlarSolyYPO+nUiy0if3XPNHtcdoUgR97CGWfrm1U+3Rb/gDN3RNDTmWveISB8B2d3j00dUvQEwdYBmK48RsJ1z0a0bjc4E3D2Ns0RiOsp4kWi+CEsMG7r0AVWpdOgusiVnx4bWgiALQJclJS6xkszB63PjIjNyscWr79KE6YFmYxP0aEOtLh/u7FAuWCWM7oFEvXp+b4gOZi+dH1SeuKpNO1Cm+6O6mR0bfnrYlrJUr8m0USh4Uoqz8Qexn8SxRtR1cFQENyDzrTqi97aInmA9ZtiaAXRDy6gByOZ9uP0SYgqdF2+R0xNGQsVF6mo7vBqDynpqMp39Bi1DfceFP2/7qLIgsiBvSEH9fj903RAg4rlra16JiQKw2jwHN/S3I0PBvkV20AlgAyFjBAcIdvVpi1OjRIOXKPKPZPB3tyc940U0z3tHSJoWDlKWv5jq1W6YvmzV+bqVpHCAoKatQlcePhKv0WwQC0sFQnfs4JgYqBrpQunSzof4l62NsOsW/9Ji7NBQnWadFmTtoyC4pXRDYitq6ZURRasQS929V9OpzAFQbH55guB6RYyRuBdfCJlQlepZOy40ZV9A+5w="
-  
+
   ### === BluemixPassword === ###
   - secure: "pCcGxYi4rP0giVfguOMDrH6nq++v5xx0geb1pBQT6C6awF6NoFWfkrZXQSGqx5/G2bP5F4G+djTP0c441bzSSPrUT+oqW47vdrg6vqYRVin/YZFQRlutoHPcckjyyWep2aa5qAbp31yWXCol3MwQPupQl90yQeXKUk4GI9HP3+h9wEKeus0x+RZvLuKVINWNlRWMaIHtnR1J8xTrzCCFqQR4L6Kknv80sxOfTMFyaEKv8dUdAHzrV87J+tYQAaJoKAsMLqdagX88TBK9uRxXmfAX33RCUIdgDlG6zorRFcWG+xEvDcBaygxayZ3lc0niqJfAxJqFwzW5ORjYowzXqlfrswpp49djcTUe+fJePLUeApR92BAVXlr12xrbhYBG9BCaOlPvE5JCtmS2BZpGtuZa3XQ1PD9+R0mu9wFz3YgvFMS1WsFhqpauSp4sTmChvpV64jhkbZtjhIymEdOOxN7r9QNetr8P2JyMT5k86TYl0p7+P1O62ru8tqycujIMVLd5KeyH08K71aNmAjdn46Ft5g3sX+rZm7L19EsFRkojTSh7QY6hXW3YuCX3pVQTR25ZqkyZKFV6aFCP0kio7MdN5VcK9iIvDA15pV/ElFNEVMiGKwkC2MFzGD+Qb6MGqE7I4ShwjdU0qLoey3+KJX5pnGYX+CA6Jz2ac5vW6YM="
 

application/celery.py

@@ -29,7 +29,6 @@ def load_env():
 from django.conf import settings
 from django.apps import AppConfig
 
-
 # set the default Django settings module for the 'celery' program.
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'application.settings')
 
@@ -48,33 +47,39 @@ def load_env():
 app.conf.broker_use_ssl             = settings.BROKER_USE_SSL
 app.conf.accept_content             = settings.CELERY_ACCEPT_CONTENT
 app.conf.timezone                   = settings.CELERY_TIMEZONE
+app.conf.enable_utc                 = settings.CELERY_ENABLE_UTC
 
 # Worker settings
-app.conf.worker_concurrency         = settings.CELERYD_CONCURRENCY
+app.conf.worker_concurrency         = settings.CELERY_CONCURRENCY
 
 # Results settings
-#app.conf.result_backend             = settings.CELERY_RESULT_BACKEND
+#app.conf.result_backend            = settings.CELERY_RESULT_BACKEND
 app.conf.result_serializer          = settings.CELERY_RESULT_SERIALIZER
 app.conf.result_expires             = settings.CELERY_TASK_RESULT_EXPIRES
 
 # Task settings
 app.conf.task_serializer            = settings.CELERY_TASK_SERIALIZER
 app.conf.task_acks_late             = settings.CELERY_TASK_ACKS_LATE
 app.conf.task_reject_on_worker_lost = settings.CELERY_TASK_REJECT_ON_WORKER_LOST
-app.conf.task_time_limit            = settings.CELERYD_TASK_TIME_LIMIT
-app.conf.task_soft_time_limit       = settings.CELERYD_TASK_SOFT_TIME_LIMIT
+app.conf.task_time_limit            = settings.CELERY_TASK_TIME_LIMIT
+app.conf.task_soft_time_limit       = settings.CELERY_TASK_SOFT_TIME_LIMIT
 app.conf.task_always_eager          = settings.CELERY_TASK_ALWAYS_EAGER
+app.conf.task_queues                = settings.CELERY_TASK_QUEUES
+
+# Event settings
+app.conf.event_queue_ttl            = settings.CELERY_EVENT_QUEUE_EXPIRES
+app.conf.event_queue_expires        = settings.CELERY_EVENT_QUEUE_TTL
 
 # Celery Beat Settings
-app.conf.beat_scheduler         = settings.CELERYBEAT_SCHEDULER
-app.conf.beat_schedule          = settings.CELERYBEAT_SCHEDULE
-app.conf.beat_sync_every        = settings.CELERYBEAT_SYNC_EVERY
-app.conf.beat_max_loop_interval = settings.CELERYBEAT_MAX_LOOP_INTERVAL
+app.conf.beat_scheduler             = settings.CELERYBEAT_SCHEDULER
+app.conf.beat_schedule              = settings.CELERYBEAT_SCHEDULE
+app.conf.beat_sync_every            = settings.CELERYBEAT_SYNC_EVERY
+app.conf.beat_max_loop_interval     = settings.CELERYBEAT_MAX_LOOP_INTERVAL
 
 # Celery Monitor Settings
-app.conf.monitors_expire_success = timedelta(hours=1)
-app.conf.monitors_expire_error   = timedelta(days=3)
-app.conf.monitors_expire_pending = timedelta(days=5)
+app.conf.monitors_expire_success    = timedelta(hours=1)
+app.conf.monitors_expire_error      = timedelta(days=3)
+app.conf.monitors_expire_pending    = timedelta(days=5)
 
 class CeleryConfig(AppConfig):
     name = 'application'

application/settings.py

@@ -19,6 +19,7 @@
 from datetime import timedelta
 
 from celery.schedules import crontab
+from kombu import Exchange, Queue
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 BASE_DIR = os.path.dirname(os.path.dirname(__file__))
@@ -119,8 +120,6 @@ def assign_env_value(var_name):
 ]
 
 THIRD_PARTY_APPS = [
-    'material',
-    'material.admin',
     'admin_view_permission',
     'django_extensions',
     'django_ses',
@@ -309,32 +308,52 @@ def assign_env_value(var_name):
 ALLOWED_HOSTS = ['*']
 
 # Celery Configuration
-BROKER_URL = assign_env_value('RABBITMQ_URL')
+if assign_env_value('USE_RABBITMQ'):
+    BROKER_URL = assign_env_value('RABBITMQ_URL')
+else:
+    BROKER_URL = assign_env_value('REDIS_URL')
+
 BROKER_USE_SSL=True
 
 CELERY_ACCEPT_CONTENT = ['application/json']
-CELERY_TIMEZONE = TIME_ZONE
+CELERY_TIMEZONE       = TIME_ZONE
+CELERY_ENABLE_UTC     = False
 
-CELERYD_CONCURRENCY = 3
-#CELERY_RESULT_BACKEND = 'django-db'
+CELERY_CONCURRENCY       = 3
+#CELERY_RESULT_BACKEND   = 'django-db'
 CELERY_RESULT_SERIALIZER = 'json'
 
-CELERY_TASK_SERIALIZER = 'json'
-CELERY_TASK_ACKS_LATE=True # Acknoledge pool when task is over
-CELERY_TASK_REJECT_ON_WORKER_LOST=True
-CELERY_TASK_RESULT_EXPIRES=7*24*30*30
+CELERY_TASK_SERIALIZER            = 'json'
+CELERY_TASK_ACKS_LATE             = True # Acknoledge pool when task is over
+CELERY_TASK_REJECT_ON_WORKER_LOST = True
+CELERY_TASK_RESULT_EXPIRES        = 3*24*60*60 # 3 Days
 
-CELERYD_TASK_TIME_LIMIT=90
-CELERYD_TASK_SOFT_TIME_LIMIT=60
+CELERY_EVENT_QUEUE_EXPIRES = 60
+CELERY_EVENT_QUEUE_TTL     = 5
+
+CELERY_TASK_TIME_LIMIT      = 90
+CELERY_TASK_SOFT_TIME_LIMIT = 60
 
 if DEBUG or TESTING:
     CELERY_TASK_ALWAYS_EAGER = True
 else:
     CELERY_TASK_ALWAYS_EAGER = False
 
-CELERYBEAT_SCHEDULER = 'django_celery_beat.schedulers:DatabaseScheduler'
-CELERYBEAT_MAX_LOOP_INTERVAL=10
-CELERYBEAT_SYNC_EVERY=1
+CELERY_TASK_QUEUES = [
+    Queue(
+        'celery',
+        Exchange('celery'),
+        routing_key = 'celery',
+        queue_arguments = {
+            'x-message-ttl': 60 * 1000 # 60 000 ms = 60 secs.
+        }
+    )
+]
+
+CELERYBEAT_SCHEDULER         = 'django_celery_beat.schedulers:DatabaseScheduler'
+CELERYBEAT_MAX_LOOP_INTERVAL = 10
+CELERYBEAT_SYNC_EVERY        = 1
+
 CELERYBEAT_SCHEDULE = {
     'refresh_all_rss_subscribers_count': {
         'task': 'feedcrunch.tasks.refresh_all_rss_subscribers_count',