Merge pull request #10 from NathanWorkman/feature/pytest

Feature/pytest

.gitignore

@@ -17,6 +17,7 @@ __pycache__
 # due to using tox and pytest
 .tox
 .cache
+.pytest_cache
 
 # node modules
 /node_modules

.travis.yml

@@ -0,0 +1,12 @@
+language: python
+
+python:
+  - "3.6"
+
+install:
+  - pip install setuptools --upgrade
+  - pip install tox-travis
+  - pip install -r requirements.txt
+
+script:
+  - tox

requirements.in

@@ -9,4 +9,10 @@ pip-tools==1.11.0
 psycopg2==2.7.4
 redis==2.10.6
 Scrapy==1.5.0
-scrapy-djangoitem==1.1.1
+scrapy-djangoitem==1.1.1
+pytest==3.5.0
+pytest-django==3.1.2
+flake8==3.5.0
+tox==3.0.0
+pytest-factoryboy==2.0.1
+

requirements.txt

@@ -6,7 +6,7 @@
 #
 amqp==2.2.2               # via kombu
 asn1crypto==0.24.0        # via cryptography
-attrs==17.4.0             # via automat, service-identity
+attrs==17.4.0             # via automat, pytest, service-identity
 automat==0.6.0            # via twisted
 billiard==3.5.0.3         # via celery
 celery==4.1.0
@@ -15,39 +15,56 @@ cffi==1.11.5              # via cryptography
 chardet==3.0.4            # via requests
 click==6.7                # via pip-tools
 constantly==15.1.0        # via twisted
-cryptography==2.2.1       # via pyopenssl
+cryptography==2.2.2       # via pyopenssl
 cssselect==1.0.3          # via parsel, scrapy
 django-anymail==2.0
 django-debug-toolbar==1.9.1
 django-dotenv==1.4.2
 django-widget-tweaks==1.4.1
 django==2.0.3
+factory-boy==2.10.0       # via pytest-factoryboy
+faker==0.8.12             # via factory-boy
 first==2.0.1              # via pip-tools
+flake8==3.5.0
 gunicorn==19.7.1
 hyperlink==18.0.0         # via twisted
 idna==2.6                 # via cryptography, hyperlink, requests
 incremental==17.5.0       # via twisted
+inflection==0.3.1         # via pytest-factoryboy
 kombu==4.1.0              # via celery
 lxml==4.2.1               # via parsel, scrapy
+mccabe==0.6.1             # via flake8
+more-itertools==4.1.0     # via pytest
 parsel==1.4.0             # via scrapy
 pip-tools==1.11.0
+pluggy==0.6.0             # via pytest, tox
 psycopg2==2.7.4
+py==1.5.3                 # via pytest, tox
 pyasn1-modules==0.2.1     # via service-identity
 pyasn1==0.4.2             # via pyasn1-modules, service-identity
+pycodestyle==2.3.1        # via flake8
 pycparser==2.18           # via cffi
 pydispatcher==2.0.5       # via scrapy
+pyflakes==1.6.0           # via flake8
 pyopenssl==17.5.0         # via scrapy, service-identity
+pytest-django==3.1.2
+pytest-factoryboy==2.0.1
+pytest==3.5.0
+python-dateutil==2.7.2    # via faker
 pytz==2018.3              # via celery, django
 queuelib==1.5.0           # via scrapy
 redis==2.10.6
 requests==2.18.4          # via django-anymail
 scrapy-djangoitem==1.1.1
 scrapy==1.5.0
 service-identity==17.0.0  # via scrapy
-six==1.11.0               # via automat, cryptography, django-anymail, parsel, pip-tools, pyopenssl, scrapy, scrapy-djangoitem, w3lib
+six==1.11.0               # via automat, cryptography, django-anymail, faker, more-itertools, parsel, pip-tools, pyopenssl, pytest, python-dateutil, scrapy, scrapy-djangoitem, tox, w3lib
 sqlparse==0.2.4           # via django-debug-toolbar
+text-unidecode==1.2       # via faker
+tox==3.0.0
 twisted==17.9.0           # via scrapy
 urllib3==1.22             # via requests
 vine==1.1.4               # via amqp
+virtualenv==15.2.0        # via tox
 w3lib==1.19.0             # via parsel, scrapy
 zope.interface==4.4.3     # via twisted

seeker/company/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class CompanyConfig(AppConfig):
+    name = 'company'

seeker/companies/migrations/0001_initial.py → seeker/company/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 2.0.3 on 2018-03-27 00:40
+# Generated by Django 2.0.3 on 2018-04-06 04:39
 
 from django.db import migrations, models
 

seeker/job/admin.py

@@ -0,0 +1,6 @@
+from django.contrib import admin
+
+from .models import Job, Board
+
+admin.site.register(Board)
+admin.site.register(Job)

seeker/job/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class JobConfig(AppConfig):
+    name = 'job'

seeker/jobs/migrations/0001_initial.py → seeker/job/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 2.0.3 on 2018-03-27 00:40
+# Generated by Django 2.0.3 on 2018-04-06 04:39
 
 from django.db import migrations, models
 import django.db.models.deletion
@@ -9,7 +9,7 @@ class Migration(migrations.Migration):
     initial = True
 
     dependencies = [
-        ('companies', '0001_initial'),
+        ('company', '0001_initial'),
     ]
 
     operations = [
@@ -36,11 +36,11 @@ class Migration(migrations.Migration):
                 ('scrape_date', models.DateTimeField()),
                 ('salary', models.TextField()),
                 ('location', models.TextField()),
-                ('board', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jobs.Board')),
-                ('company', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='companies.Company')),
+                ('board', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='job.Board')),
+                ('company', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='company.Company')),
             ],
             options={
-                'ordering': ['-pub_date'],
+                'ordering': ['-scrape_date'],
             },
         ),
     ]

seeker/jobs/models.py → seeker/job/models.py

@@ -2,7 +2,7 @@
 from django.urls import reverse
 from django.db import models
 
-from companies.models import Company
+from company.models import Company
 
 
 class Board(models.Model):
@@ -27,11 +27,11 @@ class Job(models.Model):
         on_delete=models.CASCADE
     )
     company = models.ForeignKey(
-            Company,
-            blank=True,
-            null=True,
-            on_delete=models.SET_NULL
-        )
+        Company,
+        blank=True,
+        null=True,
+        on_delete=models.SET_NULL
+    )
     body = models.TextField()
     url = models.URLField("URL")
     pub_date = models.TextField()

seeker/manage.py

@@ -11,5 +11,5 @@
             "Couldn't import Django. Are you sure it's installed and "
             "available on your PYTHONPATH environment variable? Did you "
             "forget to activate a virtual environment?"
-        ) from exc
+        )
     execute_from_command_line(sys.argv)

seeker/scraper/items.py

@@ -8,7 +8,7 @@
 import scrapy
 from scrapy_djangoitem import DjangoItem
 
-from jobs.models import Job
+from job.models import Job
 
 
 class JobItem(DjangoItem):

seeker/scraper/pipelines.py

@@ -5,15 +5,11 @@
 # Don't forget to add your pipeline to the ITEM_PIPELINES setting
 # See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
 
-import re
-from datetime import datetime
-
-import pytz
 from django.utils.text import slugify
 from scrapy.exceptions import DropItem
 
-from jobs.models import Job, Board
-from companies.models import Company
+from job.models import Job, Board
+from company.models import Company
 
 
 class JobsPipeline(object):
@@ -40,7 +36,7 @@ def process_item(self, item, spider):
         """
         For every job that we create, delete the oldest job
         """
-        # TODO: possibly remove this and just keep a large index of 
+        # TODO: possibly remove this and just keep a large index of
         # job = Job.objects.all().last()
         # job.delete()
 

seeker/scraper/settings.py

@@ -18,78 +18,78 @@
 
 
 # Crawl responsibly by identifying yourself (and your website) on the user-agent
-#USER_AGENT = 'scraper (+http://www.yourdomain.com)'
+# USER_AGENT = 'scraper (+http://www.yourdomain.com)'
 
 # Obey robots.txt rules
 ROBOTSTXT_OBEY = False
 
 # Configure maximum concurrent requests performed by Scrapy (default: 16)
-#CONCURRENT_REQUESTS = 32
+# CONCURRENT_REQUESTS = 32
 
 # Configure a delay for requests for the same website (default: 0)
 # See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
 # See also autothrottle settings and docs
-#DOWNLOAD_DELAY = 3
+# DOWNLOAD_DELAY = 3
 # The download delay setting will honor only one of:
-#CONCURRENT_REQUESTS_PER_DOMAIN = 16
-#CONCURRENT_REQUESTS_PER_IP = 16
+# CONCURRENT_REQUESTS_PER_DOMAIN = 16
+# CONCURRENT_REQUESTS_PER_IP = 16
 
 # Disable cookies (enabled by default)
-#COOKIES_ENABLED = False
+# COOKIES_ENABLED = False
 
 # Disable Telnet Console (enabled by default)
-#TELNETCONSOLE_ENABLED = False
+# TELNETCONSOLE_ENABLED = False
 
 # Override the default request headers:
-#DEFAULT_REQUEST_HEADERS = {
+# DEFAULT_REQUEST_HEADERS = {
 #   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
 #   'Accept-Language': 'en',
-#}
+# }
 
 # Enable or disable spider middlewares
 # See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
-#SPIDER_MIDDLEWARES = {
+# SPIDER_MIDDLEWARES = {
 #    'scraper.middlewares.ScraperSpiderMiddleware': 543,
-#}
+# }
 
 # Enable or disable downloader middlewares
 # See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
-#DOWNLOADER_MIDDLEWARES = {
+# DOWNLOADER_MIDDLEWARES = {
 #    'scraper.middlewares.ScraperDownloaderMiddleware': 543,
-#}
+# }
 
 # Enable or disable extensions
 # See https://doc.scrapy.org/en/latest/topics/extensions.html
-#EXTENSIONS = {
+# EXTENSIONS = {
 #    'scrapy.extensions.telnet.TelnetConsole': None,
-#}
+# }
 
 # Configure item pipelines
 # See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
 ITEM_PIPELINES = {
-   'scraper.pipelines.JobsPipeline': 300,
+    'scraper.pipelines.JobsPipeline': 300,
 }
 
 # Enable and configure the AutoThrottle extension (disabled by default)
 # See https://doc.scrapy.org/en/latest/topics/autothrottle.html
-#AUTOTHROTTLE_ENABLED = True
+# AUTOTHROTTLE_ENABLED = True
 # The initial download delay
-#AUTOTHROTTLE_START_DELAY = 5
+# AUTOTHROTTLE_START_DELAY = 5
 # The maximum download delay to be set in case of high latencies
-#AUTOTHROTTLE_MAX_DELAY = 60
+# AUTOTHROTTLE_MAX_DELAY = 60
 # The average number of requests Scrapy should be sending in parallel to
 # each remote server
-#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
+# AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
 # Enable showing throttling stats for every response received:
-#AUTOTHROTTLE_DEBUG = False
+# AUTOTHROTTLE_DEBUG = False
 
 # Enable and configure HTTP caching (disabled by default)
 # See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
-#HTTPCACHE_ENABLED = True
-#HTTPCACHE_EXPIRATION_SECS = 0
-#HTTPCACHE_DIR = 'httpcache'
-#HTTPCACHE_IGNORE_HTTP_CODES = []
-#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
+# HTTPCACHE_ENABLED = True
+# HTTPCACHE_EXPIRATION_SECS = 0
+# HTTPCACHE_DIR = 'httpcache'
+# HTTPCACHE_IGNORE_HTTP_CODES = []
+# HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
 
 
 # http://doc.scrapy.org/en/latest/topics/djangoitem.html#django-settings-set-up

seeker/seeker/settings.py

@@ -42,8 +42,8 @@
     'debug_toolbar',
     'widget_tweaks',
 
-    'jobs',
-    'companies',
+    'job',
+    'company',
 ]
 
 MIDDLEWARE = [
@@ -143,4 +143,3 @@
 MEDIAFILES_LOCATION = 'media'
 MEDIA_ROOT = os.path.join(BASE_DIR, 'media')
 MEDIA_URL = '/media/'
-

seeker/seeker/urls.py

@@ -23,6 +23,6 @@
     path('admin/', admin.site.urls),
     path('', HomeView.as_view(), name='home'),
     path('about/', AboutView.as_view(), name='about'),
-    path('jobs/', include('jobs.urls')),
-    path('companies/', include('companies.urls'))
+    path('jobs/', include('job.urls')),
+    path('companies/', include('company.urls'))
 ]