0_data_scraping.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#!/usr/bin/python\n",
    "\n",
    "#-----------------------------------------------------------------------\n",
    "# twitter-retweets\n",
    "#  - print who has retweeted tweets from a given user's timeline\n",
    "#-----------------------------------------------------------------------\n",
    "\n",
    "from twitter import *\n",
    "from pymongo import MongoClient\n",
    "import json\n",
    "import time\n",
    "\n",
    "client = MongoClient()\n",
    "db = client['twitter']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#-----------------------------------------------------------------------\n",
    "# load our API credentials \n",
    "#-----------------------------------------------------------------------\n",
    "config = {}\n",
    "execfile(\"config.py\", config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def pp_json(json_thing, sort=True, indents=4):\n",
    "    if type(json_thing) is str:\n",
    "        print(json.dumps(json.loads(json_thing), sort_keys=sort, indent=indents))\n",
    "    else:\n",
    "        print(json.dumps(json_thing, sort_keys=sort, indent=indents))\n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#-----------------------------------------------------------------------\n",
    "# create twitter API object\n",
    "#-----------------------------------------------------------------------\n",
    "twitter = Twitter(auth = OAuth(config[\"access_key\"], config[\"access_secret\"], config[\"consumer_key\"], config[\"consumer_secret\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#-----------------------------------------------------------------------\n",
    "# loop through each of my statuses, and print its content\n",
    "#-----------------------------------------------------------------------\n",
    "def get_posts(user):\n",
    "    print db.posts.find({\"user.screen_name\": user}).count()\n",
    "    prev_id = 0\n",
    "    for status in db.posts.find({\"user.screen_name\": user}).sort([(\"id\", 1)]).limit(1):\n",
    "        prev_id = status[\"id\"]\n",
    "\n",
    "    while True:\n",
    "        try:\n",
    "            if prev_id != 0:\n",
    "                results = twitter.statuses.user_timeline(screen_name=user, max_id = prev_id, count=200)\n",
    "            elif last_id != 0:\n",
    "                results = twitter.statuses.user_timeline(screen_name=user, since_id = last_id, count=200)\n",
    "            else:\n",
    "                results = twitter.statuses.user_timeline(screen_name=user, count=200)\n",
    "            for status in results:\n",
    "                #print \"@%s %s \\n%s\" % (user, status[\"id\"], status[\"text\"])\n",
    "                db.posts.insert_one(status)\n",
    "            print db.posts.find({\"user.screen_name\": user}).count()\n",
    "            prev_id = results[-1][\"id\"]\n",
    "        except:\n",
    "            print \"sleeping...\"\n",
    "            pp_json(twitter.application.rate_limit_status()[\"resources\"][\"statuses\"][\"/statuses/user_timeline\"])\n",
    "            time.sleep(60*15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#-----------------------------------------------------------------------\n",
    "# loop through each of user's statuses, and get retweeter list \n",
    "#-----------------------------------------------------------------------\n",
    "def get_retweeter_list_from_posts(user):\n",
    "    print db.posts.find({\"user.screen_name\": user}).count()\n",
    "    users = {}\n",
    "    i = 0\n",
    "    for status in db.posts.find({\"user.screen_name\": user}, {\"id\":1, \"text\":1, \"created_at\":1}, no_cursor_timeout=True):\n",
    "        if db.retweeter_list.find_one({\"tweet_id\": status[\"id\"]}):\n",
    "            continue\n",
    "        else:\n",
    "            #print \"@%s %s %s \" % (user, status[\"text\"], status[\"created_at\"]) \n",
    "            #print status[\"id\"]\n",
    "            if i%100 == 0:\n",
    "                pp_json(twitter.application.rate_limit_status()[\"resources\"][\"statuses\"][\"/statuses/retweeters/ids\"])\n",
    "                print db.retweeter_list.count()\n",
    "            i += 1\n",
    "            try:\n",
    "                retweets = twitter.statuses.retweeters.ids(_id=status[\"id\"])\n",
    "                db.retweeter_list.update({\"tweet_id\": status[\"id\"]}, {'$set': {\"retweeters\": retweets[\"ids\"]}}, upsert=True)\n",
    "            except:\n",
    "                print \"sleeping...\"\n",
    "                pp_json(twitter.application.rate_limit_status()[\"resources\"][\"statuses\"][\"/statuses/retweeters/ids\"])\n",
    "                time.sleep(60*15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#-----------------------------------------------------------------------\n",
    "# loop through retweeter list, and get the users infomation\n",
    "#-----------------------------------------------------------------------\n",
    "def get_users_from_retweeter_list():\n",
    "    i = 0\n",
    "    count = 0\n",
    "    for retweeter_list in db.retweeter_list.find({}, {\"tweet_id\":1, \"retweeters\":1}, no_cursor_timeout=True):\n",
    "        if db.users.find_one({\"tweet_id\": retweeter_list[\"tweet_id\"] }):\n",
    "            continue\n",
    "        else:\n",
    "            \n",
    "            if i%100 == 0:\n",
    "                pp_json(twitter.application.rate_limit_status()[\"resources\"][\"users\"][\"/users/lookup\"])\n",
    "            i += 1\n",
    "            try:\n",
    "                if len(retweeter_list[\"retweeters\"]) > 0:\n",
    "                    retweeters = twitter.users.lookup(user_id=retweeter_list[\"retweeters\"])\n",
    "                    count += 1\n",
    "                    for i in range(len(retweeters)):\n",
    "                        print \"@%s %s \" % (retweeters[i][\"id\"], retweeters[i][\"name\"])\n",
    "                        db.users.insert_one({\"tweet_id\": retweeter_list[\"tweet_id\"], \"user\": retweeters[i]})\n",
    "                    print \"number of retweeters: \", retweeter_list[\"tweet_id\"], len(retweeter_list[\"retweeters\"])\n",
    "                if count == 900:\n",
    "                    print \"sleeping...\"\n",
    "                    count = 0\n",
    "                    pp_json(twitter.application.rate_limit_status()[\"resources\"][\"users\"][\"/users/lookup\"])\n",
    "                    time.sleep(60*15)\n",
    "            except:\n",
    "                continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "counts = {}\n",
    "for retweeter_list in db.retweeter_list.find({}, {\"tweet_id\":1, \"retweeters\":1}, no_cursor_timeout=True):\n",
    "    name = db.posts.find_one({\"id\": retweeter_list[\"tweet_id\"]})[\"user\"][\"screen_name\"]\n",
    "    db.retweeter_list.update_one({\"tweet_id\": retweeter_list[\"tweet_id\"]}, {'$set': {\"source\": name}}, upsert=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for user in db.users.find({}, {\"tweet_id\":1}, no_cursor_timeout=True):\n",
    "    name = db.posts.find_one({\"id\": user[\"tweet_id\"]})[\"user\"][\"screen_name\"]\n",
    "    db.users.update_one({\"tweet_id\": user[\"tweet_id\"]}, {'$set': {\"source\": name}}, upsert=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import newspaper\n",
    "from newspaper import Article"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_article(url):\n",
    "    a = Article(url, language='en')\n",
    "    a.download()\n",
    "    a.parse()\n",
    "    return a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def article_to_document(article, idx):\n",
    "    document = {\"top_image\":article.top_image, \"text\": article.text, \"title\": article.title, \"id\": idx, \"authors\": article.authors}\n",
    "    document[\"images\"] = article.images\n",
    "    document[\"movies\"] = article.movies\n",
    "    article.nlp()\n",
    "    document[\"summary\"] = article.summary\n",
    "    document[\"keywords\"] = article.keywords\n",
    "    return document"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}