chromedriver and screenshot timeout.

mikeumus · Jul 15, 2015 · 9b57c92 · 9b57c92
1 parent 84f77b6
commit 9b57c92
Show file tree

Hide file tree

Showing 74 changed files with 27 additions and 5 deletions.
diff --git a/chromedriver b/chromedriver
diff --git a/notes.md b/notes.md
@@ -10,3 +10,11 @@ Postgres in Cloud 9:
  - `sudo -s` then `sudo -u postgres psql`
  - createuser -d -P -s pixm
  - createdb -O mike phishingdb
+
+Install psycopg2 for postgres
+- http://stackoverflow.com/a/5450183
+- `sudo apt-get install libpq-dev python-dev`
+- `sudo apt-get install psycopg2`
+
+Using chromedriver
+- http://selenium-python.readthedocs.org/en/latest/faq.html#how-to-use-chromedriver
diff --git a/scrapy_phishtank/phishots/AOL_3309944_screenshot.png b/scrapy_phishtank/phishots/AOL_3309944_screenshot.png
diff --git a/scrapy_phishtank/phishots/AOL_3309963_screenshot.png b/scrapy_phishtank/phishots/AOL_3309963_screenshot.png
diff --git a/scrapy_phishtank/phishots/AOL_3309973_screenshot.png b/scrapy_phishtank/phishots/AOL_3309973_screenshot.png
diff --git a/scrapy_phishtank/phishots/AOL_3309978_screenshot.png b/scrapy_phishtank/phishots/AOL_3309978_screenshot.png
diff --git a/scrapy_phishtank/phishots/AOL_3309980_screenshot.png b/scrapy_phishtank/phishots/AOL_3309980_screenshot.png
diff --git a/scrapy_phishtank/phishots/AOL_3309986_screenshot.png b/scrapy_phishtank/phishots/AOL_3309986_screenshot.png
diff --git a/scrapy_phishtank/phishots/AOL_3309987_screenshot.png b/scrapy_phishtank/phishots/AOL_3309987_screenshot.png
diff --git a/scrapy_phishtank/phishots/AOL_3310026_screenshot.png b/scrapy_phishtank/phishots/AOL_3310026_screenshot.png
diff --git a/scrapy_phishtank/phishots/AOL_3310051_screenshot.png b/scrapy_phishtank/phishots/AOL_3310051_screenshot.png
diff --git a/scrapy_phishtank/phishots/AOL_3310052_screenshot.png b/scrapy_phishtank/phishots/AOL_3310052_screenshot.png
diff --git a/scrapy_phishtank/phishots/AOL_3310061_screenshot.png b/scrapy_phishtank/phishots/AOL_3310061_screenshot.png
diff --git a/scrapy_phishtank/phishots/AOL_3310062_screenshot.png b/scrapy_phishtank/phishots/AOL_3310062_screenshot.png
diff --git a/scrapy_phishtank/phishots/AOL_3310070_screenshot.png b/scrapy_phishtank/phishots/AOL_3310070_screenshot.png
diff --git a/scrapy_phishtank/phishots/AOL_3310097_screenshot.png b/scrapy_phishtank/phishots/AOL_3310097_screenshot.png
diff --git a/scrapy_phishtank/phishots/AOL_3310111_screenshot.png b/scrapy_phishtank/phishots/AOL_3310111_screenshot.png
diff --git a/scrapy_phishtank/phishots/AOL_3310112_screenshot.png b/scrapy_phishtank/phishots/AOL_3310112_screenshot.png
diff --git a/scrapy_phishtank/phishots/AOL_3310319_screenshot.png b/scrapy_phishtank/phishots/AOL_3310319_screenshot.png
diff --git a/scrapy_phishtank/phishots/AOL_3310327_screenshot.png b/scrapy_phishtank/phishots/AOL_3310327_screenshot.png
diff --git a/scrapy_phishtank/phishots/AOL_3310328_screenshot.png b/scrapy_phishtank/phishots/AOL_3310328_screenshot.png
diff --git a/scrapy_phishtank/phishots/AOL_3310333_screenshot.png b/scrapy_phishtank/phishots/AOL_3310333_screenshot.png
diff --git a/scrapy_phishtank/phishots/AOL_3310334_screenshot.png b/scrapy_phishtank/phishots/AOL_3310334_screenshot.png
diff --git a/scrapy_phishtank/phishots/AOL_3310347_screenshot.png b/scrapy_phishtank/phishots/AOL_3310347_screenshot.png
diff --git a/scrapy_phishtank/phishots/AOL_3310348_screenshot.png b/scrapy_phishtank/phishots/AOL_3310348_screenshot.png
diff --git a/scrapy_phishtank/phishots/AOL_3310349_screenshot.png b/scrapy_phishtank/phishots/AOL_3310349_screenshot.png
diff --git a/scrapy_phishtank/phishots/AOL_3310350_screenshot.png b/scrapy_phishtank/phishots/AOL_3310350_screenshot.png
diff --git a/scrapy_phishtank/phishots/AOL_3310354_screenshot.png b/scrapy_phishtank/phishots/AOL_3310354_screenshot.png
diff --git a/scrapy_phishtank/phishots/AOL_3310355_screenshot.png b/scrapy_phishtank/phishots/AOL_3310355_screenshot.png
diff --git a/scrapy_phishtank/phishots/Apple_3310326_screenshot.png b/scrapy_phishtank/phishots/Apple_3310326_screenshot.png
diff --git a/scrapy_phishtank/phishots/Apple_3310331_screenshot.png b/scrapy_phishtank/phishots/Apple_3310331_screenshot.png
diff --git a/scrapy_phishtank/phishots/Apple_3310341_screenshot.png b/scrapy_phishtank/phishots/Apple_3310341_screenshot.png
diff --git a/scrapy_phishtank/phishots/Bradesco_3310003_screenshot.png b/scrapy_phishtank/phishots/Bradesco_3310003_screenshot.png
diff --git a/scrapy_phishtank/phishots/Bradesco_3310005_screenshot.png b/scrapy_phishtank/phishots/Bradesco_3310005_screenshot.png
diff --git a/scrapy_phishtank/phishots/Google_3310323_screenshot.png b/scrapy_phishtank/phishots/Google_3310323_screenshot.png
diff --git a/scrapy_phishtank/phishots/Google_3310324_screenshot.png b/scrapy_phishtank/phishots/Google_3310324_screenshot.png
diff --git a/scrapy_phishtank/phishots/Other_3309982_screenshot.png b/scrapy_phishtank/phishots/Other_3309982_screenshot.png
diff --git a/scrapy_phishtank/phishots/Other_3309985_screenshot.png b/scrapy_phishtank/phishots/Other_3309985_screenshot.png
diff --git a/scrapy_phishtank/phishots/Other_3309988_screenshot.png b/scrapy_phishtank/phishots/Other_3309988_screenshot.png
diff --git a/scrapy_phishtank/phishots/Other_3309989_screenshot.png b/scrapy_phishtank/phishots/Other_3309989_screenshot.png
diff --git a/scrapy_phishtank/phishots/Other_3309990_screenshot.png b/scrapy_phishtank/phishots/Other_3309990_screenshot.png
diff --git a/scrapy_phishtank/phishots/Other_3309991_screenshot.png b/scrapy_phishtank/phishots/Other_3309991_screenshot.png
diff --git a/scrapy_phishtank/phishots/Other_3309998_screenshot.png b/scrapy_phishtank/phishots/Other_3309998_screenshot.png
diff --git a/scrapy_phishtank/phishots/Other_3310032_screenshot.png b/scrapy_phishtank/phishots/Other_3310032_screenshot.png
diff --git a/scrapy_phishtank/phishots/Other_3310050_screenshot.png b/scrapy_phishtank/phishots/Other_3310050_screenshot.png
diff --git a/scrapy_phishtank/phishots/Other_3310123_screenshot.png b/scrapy_phishtank/phishots/Other_3310123_screenshot.png
diff --git a/scrapy_phishtank/phishots/Other_3310126_screenshot.png b/scrapy_phishtank/phishots/Other_3310126_screenshot.png
diff --git a/scrapy_phishtank/phishots/Other_3310127_screenshot.png b/scrapy_phishtank/phishots/Other_3310127_screenshot.png
diff --git a/scrapy_phishtank/phishots/Other_3310132_screenshot.png b/scrapy_phishtank/phishots/Other_3310132_screenshot.png
diff --git a/scrapy_phishtank/phishots/Other_3310164_screenshot.png b/scrapy_phishtank/phishots/Other_3310164_screenshot.png
diff --git a/scrapy_phishtank/phishots/Other_3310166_screenshot.png b/scrapy_phishtank/phishots/Other_3310166_screenshot.png
diff --git a/scrapy_phishtank/phishots/Other_3310168_screenshot.png b/scrapy_phishtank/phishots/Other_3310168_screenshot.png
diff --git a/scrapy_phishtank/phishots/Other_3310169_screenshot.png b/scrapy_phishtank/phishots/Other_3310169_screenshot.png
diff --git a/scrapy_phishtank/phishots/Other_3310171_screenshot.png b/scrapy_phishtank/phishots/Other_3310171_screenshot.png
diff --git a/scrapy_phishtank/phishots/Other_3310174_screenshot.png b/scrapy_phishtank/phishots/Other_3310174_screenshot.png
diff --git a/scrapy_phishtank/phishots/Other_3310176_screenshot.png b/scrapy_phishtank/phishots/Other_3310176_screenshot.png
diff --git a/scrapy_phishtank/phishots/Other_3310178_screenshot.png b/scrapy_phishtank/phishots/Other_3310178_screenshot.png
diff --git a/scrapy_phishtank/phishots/Other_3310183_screenshot.png b/scrapy_phishtank/phishots/Other_3310183_screenshot.png
diff --git a/scrapy_phishtank/phishots/Other_3310186_screenshot.png b/scrapy_phishtank/phishots/Other_3310186_screenshot.png
diff --git a/scrapy_phishtank/phishots/Other_3310196_screenshot.png b/scrapy_phishtank/phishots/Other_3310196_screenshot.png
diff --git a/scrapy_phishtank/phishots/Other_3310225_screenshot.png b/scrapy_phishtank/phishots/Other_3310225_screenshot.png
diff --git a/scrapy_phishtank/phishots/Other_3310226_screenshot.png b/scrapy_phishtank/phishots/Other_3310226_screenshot.png
diff --git a/scrapy_phishtank/phishots/Other_3310242_screenshot.png b/scrapy_phishtank/phishots/Other_3310242_screenshot.png
diff --git a/scrapy_phishtank/phishots/Other_3310253_screenshot.png b/scrapy_phishtank/phishots/Other_3310253_screenshot.png
diff --git a/scrapy_phishtank/phishots/Other_3310272_screenshot.png b/scrapy_phishtank/phishots/Other_3310272_screenshot.png
diff --git a/scrapy_phishtank/phishots/Other_3310317_screenshot.png b/scrapy_phishtank/phishots/Other_3310317_screenshot.png
diff --git a/scrapy_phishtank/phishots/Other_3310339_screenshot.png b/scrapy_phishtank/phishots/Other_3310339_screenshot.png
diff --git a/scrapy_phishtank/phishots/Other_3310344_screenshot.png b/scrapy_phishtank/phishots/Other_3310344_screenshot.png
diff --git a/scrapy_phishtank/phishots/Other_3310374_screenshot.png b/scrapy_phishtank/phishots/Other_3310374_screenshot.png
diff --git a/scrapy_phishtank/phishots/Other_3310407_screenshot.png b/scrapy_phishtank/phishots/Other_3310407_screenshot.png
diff --git a/scrapy_phishtank/phishots/Other_3310408_screenshot.png b/scrapy_phishtank/phishots/Other_3310408_screenshot.png
diff --git a/scrapy_phishtank/phishots/Other_3310411_screenshot.png b/scrapy_phishtank/phishots/Other_3310411_screenshot.png
diff --git a/scrapy_phishtank/phishots/Smile Bank_3310409_screenshot.png b/scrapy_phishtank/phishots/Smile Bank_3310409_screenshot.png
diff --git a/scrapy_phishtank/pishing/tasks.py b/scrapy_phishtank/pishing/tasks.py
@@ -19,7 +19,6 @@
 # Phishtank API key: http://data.phishtank.com/data/{}/online-valid.json.bz2'.format(PHISHTANK_API_KEY)
 url = 'http://data.phishtank.com/data/{}/online-valid.json'.format(PHISHTANK_API_KEY)
 
-
 @shared_task(name="StorePhishURL")
 def StorePhishURL():
     r = requests.get(url)
@@ -34,27 +33,42 @@ def StorePhishURL():
         if created:
             # freshphish.fresh_id = threat['phish_id']
             freshphish.fresh_target = threat['target']
-            
+
             try:
                 # take screenshot if status code is good
                 if r.status_code != requests.codes.ok:
                     return "Error: Unexpected response {}".format(r)
                 # take screenshot of fresh_url
-                driver = webdriver.Firefox()
+                #driver = webdriver.Firefox()
+                #executable_path="/usr/bin/chromedriver"
+                driver = webdriver.Chrome()
+                # we dont have driver_arguments varialble!!!
+                # selenium just uses the webdriver APIs so why could we use driver_arguments ?
+                # python will raise an error asking where did you come with that varialble XD
+                # I see. but.. I bet it's posible somehow lol
+                # lol coding is so logical not feeling XD
+                # :p
+                # lets try ipython and we call selenium inside of it and th chromderiver too
+                # good idea, why isn't ipython my default when I do `./manage.py shell` on this tower?
+                # becasue you dont have ipython and it runs python interprester :D
+                # :p
+
+                #driver_arguments['chrome_options'].add_argument('--no-sandbox')
+                deriver.set_page_load_timeout(30)
                 driver.get(threat['url'])
                 # save "phishshots" (screenshots) in folder http://stackoverflow.com/questions/27874002/python-selenium-save-screenshot-in-newly-created-folder
                 driver.save_screenshot('phishots/{0}_{1}_screenshot.png'.format(threat['target'], threat['phish_id']))
                 driver.quit()
             except requests.exceptions.RequestException as e:
                 # A serious problem happened, like an SSLError or InvalidURL
                 return "Error: {}".format(e)
-                
+
             if threat['online'] == 'yes':
                 freshphish.live_phish = True   # u'online': u'yes'
             # Consider any status other than 2xx an error
             else:
                 freshphish.live_phish = False
-                
+
             freshphish.verification_time = threat['verification_time']
             if threat['details']:
                 freshphish.country = threat['details'][0]['country']